Question

鉴于代码在Lucene 3.0.1下运行

import java.io.*;
import org.apache.lucene.analysis.*;
import org.apache.lucene.util.Version;

public class MyAnalyzer extends Analyzer {

   public TokenStream tokenStream(String fieldName, Reader reader) {
       return
               new StopFilter(
                       true,
                       new StandardTokenizer(Version.LUCENE_30, reader),
                       StopAnalyzer.ENGLISH_STOP_WORDS_SET
               );
   }

   private static void printTokens(String string) throws IOException {
       TokenStream ts = new MyAnalyzer().tokenStream("default", new
StringReader(string));
       TermAttribute termAtt = ts.getAttribute(TermAttribute.class);
       while(ts.incrementToken()) {
           System.out.print(termAtt.term());
           System.out.print(" ");
       }
       System.out.println();
   }

   public static void main(String[] args) throws IOException {
       printTokens("one_two_three");           // prints "one two three"
       printTokens("four4_five5_six6");        // prints "four4_five5_six6"
       printTokens("seven7_eight_nine");       // prints "seven7_eight nine"
       printTokens("ten_eleven11_twelve");     // prints "ten_eleven11_twelve"
   }
}

我可以理解为什么one_two_three和four4_five5_six6被标记为原样，因为StandardTokenizer class header Javadoc中对此进行了解释。但另外两个案例更加微妙，我是我不太确定我的想法。

Q1：如果7之后seven的外观与eight成为联合令牌但与nine分开，为什么ten粘贴到eleven11？

Q2：是否有任何标准和/或简单方法可以永久地制作StandardTokenizer 分裂在下划线上？

Answer 1

这是一个有趣的发现。我不确定如何解释为什么它会为Q1做到这一点。但是，我可以提供代码来分割Q2的剩余下划线：

public class MyAnalyzer extends Analyzer {
    public TokenStream tokenStream(String fieldName, Reader reader) {
        StandardTokenizer tokenizer = new StandardTokenizer(
                Version.LUCENE_30, reader);
        TokenStream tokenStream = new StandardFilter(tokenizer);
        tokenStream = new MyTokenFilter(tokenStream);
        tokenStream = new StopFilter(true, tokenStream,
                StopAnalyzer.ENGLISH_STOP_WORDS_SET);
        return tokenStream;
    }
}

public class MyTokenFilter extends TokenFilter {
    private final TermAttribute termAttr;
    private String[] terms;
    private int pos;

    public MyTokenFilter(TokenStream tokenStream) {
        super(tokenStream);
        this.termAttr = input.addAttribute(TermAttribute.class);
    }

    public boolean incrementToken() throws IOException {
        if (terms == null) {
            if (!input.incrementToken()) {
                return false;
            }
            terms = termAttr.term().split("_");
        }

        termAttr.setTermBuffer(terms[pos++]);
        if (pos == terms.length) {
            terms = null;
            pos = 0;
        }
        return true;
    }
}

StandardTokenizer行为

1 个答案: