Question

在我的代码示例中，我在lucene索引中创建了三个文档。其中两个没有存储字段LASTNAME，但是存储了termvector，其中一个没有存储。使用LUKE，我能够遍历此字段中的所有术语（LASTNAME）。在我的代码示例中，遍历TermFreqVectors，对于存储了TermVectors的文档，它可以正常工作。

我怎样才能获得所有这些非存储条款？ LUKE是怎么做到的？

我原来的问题是，我想扩展一个大索引（60GB），其中有近100个字段与另一个字段，而无需从头开始重新创建索引，因为我们的db-setup需要40个并行计算服务器一对天。读取索引中的所有数据非常快，只需将这个新字段添加到所有存储的文档中。

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.TermFreqVector;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.LuceneTestCase;

import java.io.File;
import java.io.IOException;
import java.util.Arrays;


public class TestDocTerms extends LuceneTestCase {

public void testDocTerms() throws IOException, ParseException {
    Analyzer analyzer = new MockAnalyzer(random);

    String fieldF = "FIRSTNAME";
    String fieldL = "LASTNAME";

    // To store an index on disk, use this instead:
    Directory directory = NIOFSDirectory.open(new File("/tmp/_index_tester/"));
    RandomIndexWriter iwriter = new RandomIndexWriter(random, directory, analyzer);
    iwriter.w.setInfoStream(VERBOSE ? System.out : null);
    Document doc = new Document();
    doc.add(newField(fieldF, "Alex", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Miller", Field.Store.NO,Field.Index.ANALYZED,Field.TermVector.YES));
    iwriter.addDocument(doc);
    doc = new Document();
    doc.add(newField(fieldF, "Chris", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Smith", Field.Store.NO, Field.Index.ANALYZED));
    iwriter.addDocument(doc);
    doc = new Document();
    doc.add(newField(fieldF, "Alex", Field.Store.YES, Field.Index.ANALYZED));
    doc.add(newField(fieldL, "Beatle", Field.Store.NO, Field.Index.ANALYZED,Field.TermVector.YES));
    iwriter.addDocument(doc);
    iwriter.close();

    // Now search the index:
    IndexSearcher isearcher = new IndexSearcher(directory, true); // read-only=true
    QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, fieldF, analyzer);
    Query query = parser.parse(fieldF + ":" + "Alex");
    TopDocs hits = isearcher.search(query, null, 2);
    assertEquals(2, hits.totalHits);
    // Iterate through the results:
    for (int i = 0; i < hits.scoreDocs.length; i++) {
        Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
        assertEquals("Alex", hitDoc.get(fieldF));
        System.out.println("query for:" +query.toString()+ " with this results firstN:" + hitDoc.get(fieldF) + " and lastN:" + hitDoc.get(fieldL));
    }
    parser = new QueryParser(TEST_VERSION_CURRENT, fieldL, analyzer);
    query = parser.parse(fieldL + ":" + "Miller");
    hits = isearcher.search(query, null, 2);
    assertEquals(1, hits.totalHits);
    // Iterate through the results:
    for (int i = 0; i < hits.scoreDocs.length; i++) {
        Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
        assertEquals("Alex", hitDoc.get(fieldF));
        System.out.println("query for:" + query.toString() + " with this results firstN:" +hitDoc.get(fieldF)+ " and lastN:" +hitDoc.get(fieldL));
    }
    isearcher.close();

    // examine terms
    IndexReader ireader = IndexReader.open(directory, true); // read-only=true
    int numDocs = ireader.numDocs();

    for (int i = 0; i < numDocs; i++) {
        doc = ireader.document(i);
        System.out.println("docNum:" + i + " with:" + doc.toString());
        TermFreqVector t = ireader.getTermFreqVector(i, fieldL);
        if (t != null){
            System.out.println("Field:" + fieldL + " contains terms:" + t.toString());
        }
        TermFreqVector[] termFreqVectors = ireader.getTermFreqVectors(i);
        if (termFreqVectors != null){
            for (TermFreqVector tfv : termFreqVectors){
                String[] terms = tfv.getTerms();
                String field = tfv.getField();
                System.out.println("Field:" +field+ " contains terms:" + Arrays.toString(terms));
            }
        }
    }
    ireader.close();
}


}

Answer 1

重建未存储的文档必然是最好的努力。您通常无法撤消分析仪对值所做的更改。

当TermVectors不可用时，Luke会枚举与该字段关联的术语。这可能不尊重术语的顺序或任何格式。但这可能既不在这里也不在那里。我不知道你的newField方法究竟做了什么，但我怀疑它的默认值不是Field.TermVector.NO。

如果您想了解更多实施细节，我会抓住Luke源代码，并阅读org.getopt.luke.DocReconstructor

LUCENE：如何通过docNr获取给定doc的所有术语而不存储数据，也不知道TermVector（LUKE能够显示这个，怎么做？）

1 个答案: