Question

我正在通过Stanford Corenlp注释大量字符串作为CoreDocuments。 StanfordCoreNLP管道具有用于多线程注释的内部功能，以优化流程，但是据我所见，CoreDocument对象在我运行的版本中无法使用该功能，即stanford-corenlp-full-2018-10-05。

由于我无法使CoreDocuments的Pipelines Annotate集合，所以我尝试通过将单个注释放在多线程方法中来优化单个注释。我在多线程环境中没有问题。我按预期收到了所有结果，我唯一的缺点是时间消耗。我尝试了大约7种不同的实现，而这是最快的3种：

//ForkJoinPool is initialized in the main method in my application
private static ForkJoinPool executor = new ForkJoinPool(Runtime.getRuntime().availableProcessors(), ForkJoinPool.defaultForkJoinWorkerThreadFactory, null, false);

   public static ConcurrentMap<String, CoreDocument> getMultipleCoreDocumentsWay1(Collection<String> str) {
        ConcurrentMap<String, CoreDocument> pipelineCoreDocumentAnnotations = new MapMaker().concurrencyLevel(2).makeMap();
        str.parallelStream().forEach((str1) -> {
            CoreDocument coreDocument = new CoreDocument(str1);
            pipeline.annotate(coreDocument);
            pipelineCoreDocumentAnnotations.put(str1, coreDocument);
            System.out.println("pipelineCoreDocumentAnnotations size1: " + pipelineCoreDocumentAnnotations.size() + "\nstr size: " + str.size() + "\n");
        });
        return pipelineCoreDocumentAnnotations;
    }


     public static ConcurrentMap<String, CoreDocument> getMultipleCoreDocumentsWay4(Collection<String> str) {
        ConcurrentMap<String, CoreDocument> pipelineCoreDocumentAnnotations = new MapMaker().concurrencyLevel(2).makeMap();
        str.parallelStream().forEach((str1) -> {
            try {
                ForkJoinTask<CoreDocument> forkCD = new RecursiveTask() {
                    @Override
                    protected CoreDocument compute() {
                        CoreDocument coreDocument = new CoreDocument(str1);
                        pipeline.annotate(coreDocument);
                        return coreDocument;
                    }
                };
                forkCD.invoke();
                pipelineCoreDocumentAnnotations.put(str1, forkCD.get());
                System.out.println("pipelineCoreDocumentAnnotations2 size: " + pipelineCoreDocumentAnnotations.size() + "\nstr size: " + str.size() + "\n");
            } catch (InterruptedException | ExecutionException ex) {
                Logger.getLogger(Parsertest.class.getName()).log(Level.SEVERE, null, ex);
            }
        });
        return pipelineCoreDocumentAnnotations;
    }

    public static ConcurrentMap<String, CoreDocument> getMultipleCoreDocumentsWay7(ConcurrentMap<Integer, String> hlstatsSTR) {
        RecursiveDocumentAnnotation recursiveAnnotation = new RecursiveDocumentAnnotation(hlstatsSTR, pipeline);
        ConcurrentMap<String, CoreDocument> returnMap = new MapMaker().concurrencyLevel(2).makeMap();
        executor.execute(recursiveAnnotation);
        try {
            returnMap = recursiveAnnotation.get();
        } catch (InterruptedException | ExecutionException ex) {
            Logger.getLogger(Parsertest.class.getName()).log(Level.SEVERE, null, ex);
        }
        System.out.println("reached end\n");
        return returnMap;
    }
RecursiveDocumentAnnotation class:

    public class RecursiveDocumentAnnotation extends RecursiveTask<ConcurrentMap<String, CoreDocument>> {

    private String str;
    private StanfordCoreNLP nlp;
    private static ConcurrentMap<String, CoreDocument> pipelineCoreDocumentAnnotations;
    private static ConcurrentMap<Integer, String> hlstatsStrMap;

    public static ConcurrentMap<String, CoreDocument> getPipelineCoreDocumentAnnotations() {
        return pipelineCoreDocumentAnnotations;
    }

    public RecursiveDocumentAnnotation(ConcurrentMap<Integer, String> hlstatsStrMap, StanfordCoreNLP pipeline) {
        this.pipelineCoreDocumentAnnotations = new MapMaker().concurrencyLevel(2).makeMap();
        this.str = hlstatsStrMap.get(0);
        this.nlp = pipeline;
        this.hlstatsStrMap = hlstatsStrMap;
    }

    public RecursiveDocumentAnnotation(ConcurrentMap<Integer, String> hlstatsStrMap, StanfordCoreNLP pipeline,
            ConcurrentMap<String, CoreDocument> returnMap) {
        this.str = hlstatsStrMap.get(returnMap.size());
        this.nlp = pipeline;
        this.hlstatsStrMap = hlstatsStrMap;
        this.pipelineCoreDocumentAnnotations = returnMap;
    }

    @Override
    protected ConcurrentMap<String, CoreDocument> compute() {
        CoreDocument coreDocument = new CoreDocument(str);
        nlp.annotate(coreDocument);
        pipelineCoreDocumentAnnotations.put(str, coreDocument);
        System.out.println("hlstatsStrMap size: " + hlstatsStrMap.size() + "\npipelineCoreDocumentAnnotations size: " + pipelineCoreDocumentAnnotations.size()
                + "\n");
        if (pipelineCoreDocumentAnnotations.size() >= hlstatsStrMap.size()) {
            return pipelineCoreDocumentAnnotations;
        }
        RecursiveDocumentAnnotation recursiveAnnotation = new RecursiveDocumentAnnotation(hlstatsStrMap, nlp, pipelineCoreDocumentAnnotations);
        recursiveAnnotation.fork();
        return recursiveAnnotation.join();
    }    }

时间并行1：336562毫秒。

时间并行4：391556毫秒。

时间并行7：491639毫秒。

如果管道本身可以以某种方式进行多重注释，那么最真诚的是最大的，但是，只要我不知道如何实现此目的，我希望有人可以亲自解释一下如何分别优化CoreDocument注释。 PS：将所有字符串混在一起到单个coredocument中进行注解也不是我想要的，因为之后需要单独的Coredocuments进行组合。

Answer 1

我没有计时，但是您可以尝试以下示例代码（将测试字符串添加到字符串列表中）...它应该同时在4个文档上起作用：

package edu.stanford.nlp.examples;

import edu.stanford.nlp.pipeline.*;

import java.util.*;
import java.util.function.*;
import java.util.stream.*;


public class MultiThreadStringExample {

    public static class AnnotationCollector<T> implements Consumer<T> {

        List<T> annotations = new ArrayList<T>();

        public void accept(T ann) {
            annotations.add(ann);
        }
    }

    public static void main(String[] args) throws Exception {
        Properties props = new Properties();
        props.setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,depparse");
        props.setProperty("threads", "4");
        StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
        AnnotationCollector<Annotation> annCollector = new AnnotationCollector<Annotation>();
        List<String> exampleStrings = new ArrayList<String>();
        for (String exampleString : exampleStrings) {
            pipeline.annotate(new Annotation(exampleString), annCollector);
        }
        Thread.sleep(10000);
        List<CoreDocument> coreDocs =
                annCollector.annotations.stream().map(ann -> new CoreDocument(ann)).collect(Collectors.toList());
        for (CoreDocument coreDoc : coreDocs) {
            System.out.println(coreDoc.tokens());
        }
    }

}

如何更有效地注释多个Stanford CoreNLP CoreDocument？

1 个答案: