自然语言中断的Split String

时间:2014-04-29 00:50:54

标签: java regex string

概述

我将字符串发送到文本到语音服务器,该服务器最多可接受300个字符。由于网络延迟,返回的每个语音段之间可能会有延迟,因此我希望在最自然的暂停状态下中断语音。尽可能。

每个服务器请求都要花费我的钱,所以理想情况下我会发送最长的字符串,最多允许的字符数。

以下是我目前的实施:

private static final boolean DEBUG = true;

private static final int MAX_UTTERANCE_LENGTH = 298;
private static final int MIN_UTTERANCE_LENGTH = 200;

private static final String FULL_STOP_SPACE = ". ";
private static final String QUESTION_MARK_SPACE = "? ";
private static final String EXCLAMATION_MARK_SPACE = "! ";
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
private static final String COMMA_SPACE = ", ";
private static final String JUST_A_SPACE = " ";

public static ArrayList<String> splitUtteranceNaturalBreaks(String utterance) {

    final long then = System.nanoTime();

    final ArrayList<String> speakableUtterances = new ArrayList<String>();

    int splitLocation = 0;
    String success = null;

    while (utterance.length() > MAX_UTTERANCE_LENGTH) {

        splitLocation = utterance.lastIndexOf(FULL_STOP_SPACE, MAX_UTTERANCE_LENGTH);

        if (DEBUG) {
            System.out.println("(0 FULL STOP) - last index at: " + splitLocation);
        }

        if (splitLocation < MIN_UTTERANCE_LENGTH) {
            if (DEBUG) {
                System.out.println("(1 FULL STOP) - NOT_OK");
            }

            splitLocation = utterance.lastIndexOf(QUESTION_MARK_SPACE, MAX_UTTERANCE_LENGTH);

            if (DEBUG) {
                System.out.println("(1 QUESTION MARK) - last index at: " + splitLocation);
            }

            if (splitLocation < MIN_UTTERANCE_LENGTH) {
                if (DEBUG) {
                    System.out.println("(2 QUESTION MARK) - NOT_OK");
                }

                splitLocation = utterance.lastIndexOf(EXCLAMATION_MARK_SPACE, MAX_UTTERANCE_LENGTH);

                if (DEBUG) {
                    System.out.println("(2 EXCLAMATION MARK) - last index at: " + splitLocation);
                }

                if (splitLocation < MIN_UTTERANCE_LENGTH) {
                    if (DEBUG) {
                        System.out.println("(3 EXCLAMATION MARK) - NOT_OK");
                    }

                    splitLocation = utterance.lastIndexOf(LINE_SEPARATOR, MAX_UTTERANCE_LENGTH);

                    if (DEBUG) {
                        System.out.println("(3 SEPARATOR) - last index at: " + splitLocation);
                    }

                    if (splitLocation < MIN_UTTERANCE_LENGTH) {
                        if (DEBUG) {
                            System.out.println("(4 SEPARATOR) - NOT_OK");
                        }

                        splitLocation = utterance.lastIndexOf(COMMA_SPACE, MAX_UTTERANCE_LENGTH);

                        if (DEBUG) {
                            System.out.println("(4 COMMA) - last index at: " + splitLocation);
                        }

                        if (splitLocation < MIN_UTTERANCE_LENGTH) {
                            if (DEBUG) {
                                System.out.println("(5 COMMA) - NOT_OK");
                            }

                            splitLocation = utterance.lastIndexOf(JUST_A_SPACE, MAX_UTTERANCE_LENGTH);

                            if (DEBUG) {
                                System.out.println("(5 SPACE) - last index at: " + splitLocation);
                            }

                            if (splitLocation < MIN_UTTERANCE_LENGTH) {
                                if (DEBUG) {
                                    System.out.println("(6 SPACE) - NOT_OK");
                                }

                                splitLocation = MAX_UTTERANCE_LENGTH;

                                if (DEBUG) {
                                    System.out.println("(6 MAX_UTTERANCE_LENGTH) - last index at: " + splitLocation);
                                }

                            } else {
                                if (DEBUG) {
                                    System.out.println("Accepted");
                                }

                                splitLocation -= 1;
                            }
                        }
                    } else {
                        if (DEBUG) {
                            System.out.println("Accepted");
                        }

                        splitLocation -= 1;
                    }
                } else {
                    if (DEBUG) {
                        System.out.println("Accepted");
                    }
                }
            } else {
                if (DEBUG) {
                    System.out.println("Accepted");
                }
            }
        } else {
            if (DEBUG) {
                System.out.println("Accepted");
            }
        }

        success = utterance.substring(0, (splitLocation + 2));

        speakableUtterances.add(success.trim());

        if (DEBUG) {
            System.out.println("Split - Length: " + success.length() + " -:- " + success);
            System.out.println("------------------------------");
        }

        utterance = utterance.substring((splitLocation + 2)).trim();
    }

    speakableUtterances.add(utterance);

    if (DEBUG) {

        System.out.println("Split - Length: " + utterance.length() + " -:- " + utterance);

        final long now = System.nanoTime();
        final long elapsed = now - then;

        System.out.println("ELAPSED: " + TimeUnit.MILLISECONDS.convert(elapsed, TimeUnit.NANOSECONDS));

    }

    return speakableUtterances;
}

由于无法在lastIndexOf内使用正则表达式而导致丑陋。丑陋的,它实际上非常快。

问题

理想情况下,我想使用正则表达式,允许我的首选分隔符匹配:

private static final String firstChoice = "[.!?" + LINE_SEPARATOR + "]\\s+";
private static final Pattern pFirstChoice = Pattern.compile(firstChoice);

然后使用匹配器解决问题:

    Matcher matcher = pFirstChoice.matcher(input);

    if (matcher.find()) {
        splitLocation = matcher.start();
    }

我当前实现中的另一种选择是存储每个分隔符的位置,然后选择最近的MAX_UTTERANCE_LENGTH

我尝试了各种方法来应用MIN_UTTERANCE_LENGTH&amp; MAX_UTTERANCE_LENGTH到模式,所以它只捕获这些值并使用lookarounds来反转迭代?<=,但这是我的知识开始让我失望的地方:

private static final String poorEffort = "([.!?]{200, 298})\\s+");

最后

我想知道你们中的任何一位正则表达者是否可以达到我所追求的目标并确认实际上是否会更有效率?

我提前感谢你。

参考文献:

2 个答案:

答案 0 :(得分:1)

我会做这样的事情:

Pattern p = Pattern.compile(".{1,299}(?:[.!?]\\s+|\\n|$)", Pattern.DOTALL);
Matcher matcher = p.matcher(text);
while (matcher.find()) {
    speakableUtterances.add(matcher.group().trim());
}

正则表达式的解释:

.{1,299}                 any character between 1 and 299 times (matching the most amount possible)
(?:[.!?]\\s+|\\n|$)      followed by either .!? and whitespaces, a newline or the end of the string

您可以考虑将标点符号扩展为\p{Punct},请参阅Pattern的javadoc。

您可以在ideone上看到工作样本。

答案 1 :(得分:0)

Unicode standard定义了如何将文本分成句子和其他逻辑组件。这是一些有效的伪代码:

// tests two consecutive codepoints within the text to detect the end of sentences
boolean continueSentence(Text text, Range range1, Range range2) {
    Code code1 = text.code(range1), code2 = text.code(range2);

    // 0.2  sot ÷   
    if (code1.isStartOfText())
        return false;

    // 0.3      ÷    eot
    if (code2.isEndOfText())
        return false;

    // 3.0  CR  ×    LF
    if (code1.isCR() && code2.isLF())
        return true;

    // 4.0  (Sep | CR | LF) ÷   
    if (code1.isSep() || code1.isCR() || code1.isLF())
        return false;

    // 5.0      ×    [Format Extend]
    if (code2.isFormat() || code2.isExtend())
        return true;

    // 6.0  ATerm   ×    Numeric
    if (code1.isATerm() && (code2.isDigit() || code2.isDecimal() || code2.isNumeric()))
        return true;

    // 7.0  Upper ATerm ×    Upper
    if (code2.isUppercase() && code1.isATerm()) {
        Range range = text.previousCode(range1);
        if (range.isValid() && text.code(range).isUppercase())
            return true;
    }

    boolean allow_STerm = true, return_value = true;

    // 8.0  ATerm Close* Sp*    ×    [^ OLetter Upper Lower Sep CR LF STerm ATerm]* Lower
    Range range = range2;
    Code code = code2;
    while (!code.isOLetter() && !code.isUppercase() && !code.isLowercase() && !code.isSep() && !code.isCR() && !code.isLF() && !code.isSTerm() && !code.isATerm()) {
        if (!(range = text.nextCode(range)).isValid())
            break;
        code = text.code(range);
    }
    range = range1;
    if (code.isLowercase()) {
        code = code1;
        allow_STerm = true;
        goto Sp_Close_ATerm;
    }
    code = code1;

    // 8.1  (STerm | ATerm) Close* Sp*  ×    (SContinue | STerm | ATerm)
    if (code2.isSContinue() || code2.isSTerm() || code2.isATerm())
        goto Sp_Close_ATerm;

    // 9.0  ( STerm | ATerm ) Close*    ×    ( Close | Sp | Sep | CR | LF )
    if (code2.isClose())
        goto Close_ATerm;

    // 10.0 ( STerm | ATerm ) Close* Sp*    ×    ( Sp | Sep | CR | LF )
    if (code2.isSp() || code2.isSep() || code2.isCR() || code2.isLF())
        goto Sp_Close_ATerm;

    // 11.0 ( STerm | ATerm ) Close* Sp* (Sep | CR | LF)?   ÷   
    return_value = false;

    // allow Sep, CR, or LF zero or one times
    for (int iteration = 1; iteration != 0; iteration--) {
        if (!code.isSep() && !code.isCR() && !code.isLF()) goto Sp_Close_ATerm;
        if (!(range = text.previousCode(range)).isValid()) goto Sp_Close_ATerm;
        code = text.code(range);
    }

Sp_Close_ATerm:
    // allow zero or more Sp
    while (code.isSp() && (range = text.previousCode(range)).isValid())
        code = text.code(range);

Close_ATerm:
    // allow zero or more Close
    while (code.isClose() && (range = text.previousCode(range)).isValid())
        code = text.code(range);

    // require STerm or ATerm
    if (code.isATerm() || (allow_STerm && code.isSTerm()))
        return return_value;

    // 12.0     ×    Any
    return true;
}

然后你可以迭代这样的句子:

// pass in a range of (0, 0) to get the range of the first sentence
// returns a range with a length of 0 if there are no more sentences
Range nextSentence(Text text, Range range) {
try_again:
    range = text.nextCode(new Range(range.start + range.length, 0));
    if (!range.isValid())
        return range;
    Range next = text.nextCode(range);
    long start = range.start;
    while (next.isValid()) && text.continueSentence(range, next))
        next = text.nextCode(range = next);
    range = new Range(start, range.start + range.length - start);

    Range range2 = text.trimRange(range);
    if (!range2.isValid())
        goto try_again;

    return range2;
}

其中:

  • 范围定义为&gt; = start和&lt;的范围开始+长度
  • text.trimRange删除空格字符(可选)
  • 所有Code.is [Type]函数都是Unicode character database的查找。例如,您会在某些文件中看到某些代码点被定义为“CR”,“Sep”,“StartOfText”等。
  • Text.code(range)解码range.start文本中的代码点。不使用长度。
  • Text.nextCode和Text.previousCode根据当前代码点的范围返回字符串中下一个或上一个代码点的范围。如果该方向上没有代码点,则返回无效范围,该范围是长度为0的范围。

该标准还定义了迭代wordslinescharacters的方法。