Question

所以我正在使用matcher类，并使用它来识别我在枚举中定义的标记。

  public static enum TokenType {
    // Definitions of accepted tokens
     IF("if"), WHILE("while"), PRINT("print"), TYPE("int|string|boolean"), BOOLOP("==|!="), BOOLVAL("false|true"), INTOP("[+]"), CHAR("[a-z]"), DIGIT("[0-9]"), WHITESPACE("[ \t\f\r\n]+"), LPAREN("[(]"), RPAREN("[)]");
    public final String pattern;

    private TokenType(String pattern) {
      this.pattern = pattern;
    }
  }

  public static class Token {
    public TokenType type;
    public String data;

我的问题是，我还需要识别出模式中未定义的任何内容，并在发生错误时打印错误。

这就是我的匹配器逻辑

    // Begin matching tokens
    Matcher matcher = tokenPatterns.matcher(input);
      while (matcher.find()) {
        if (matcher.group(TokenType.DIGIT.name()) != null) {
          tokens.add(new Token(TokenType.DIGIT, matcher.group(TokenType.DIGIT.name())));
          continue;
      } else if (matcher.group(TokenType.IF.name()) != null) {
          tokens.add(new Token(TokenType.IF, matcher.group(TokenType.IF.name())));
          continue;
      } else if (matcher.group(TokenType.WHILE.name()) != null) {
          tokens.add(new Token(TokenType.WHILE, matcher.group(TokenType.WHILE.name())));
          continue;
      } else if (matcher.group(TokenType.TYPE.name()) != null) {
          tokens.add(new Token(TokenType.TYPE, matcher.group(TokenType.TYPE.name())));
          continue;
      } else if (matcher.group(TokenType.PRINT.name()) != null) {
          tokens.add(new Token(TokenType.PRINT, matcher.group(TokenType.PRINT.name())));
          continue;
      } else if (matcher.group(TokenType.BOOLOP.name()) != null) {
          tokens.add(new Token(TokenType.BOOLOP, matcher.group(TokenType.BOOLOP.name())));
          continue;
      } else if (matcher.group(TokenType.BOOLVAL.name()) != null) {
          tokens.add(new Token(TokenType.BOOLVAL, matcher.group(TokenType.BOOLVAL.name())));
          continue;
      } else if (matcher.group(TokenType.INTOP.name()) != null) {
          tokens.add(new Token(TokenType.INTOP, matcher.group(TokenType.INTOP.name())));
          continue;
      } else if (matcher.group(TokenType.CHAR.name()) != null) {
        tokens.add(new Token(TokenType.CHAR, matcher.group(TokenType.CHAR.name())));
        continue;
      } else if (matcher.group(TokenType.LPAREN.name()) != null) {
          tokens.add(new Token(TokenType.LPAREN, matcher.group(TokenType.LPAREN.name())));
          continue;
      } else if (matcher.group(TokenType.RPAREN.name()) != null) {
          tokens.add(new Token(TokenType.RPAREN, matcher.group(TokenType.RPAREN.name())));
          continue;  
      } else if (matcher.group(TokenType.WHITESPACE.name()) != null) {
          continue; 
      }
    }

    return tokens;
  }

一个可能的解决方案是在我的模式中添加一个案例，以说明尚未定义的所有内容，这看起来像是WHITESPACE（“ [\ t \ f \ r \ n] +”），LPAREN （“ [[]”）， ERROR（“ @ |＃，$，％，^，＆.....”），但我不确定实现该目标的任何现实方法。

感谢您的帮助。这是完整代码的链接，以防万一我遗漏了任何内容-https://pastebin.com/jLtnJwgj

Answer 1

您尝试这样的事情吗？

public class Lexer {

public static enum TokenType {
    // Definitions of accepted tokens
    IF("if"),
    WHILE("while"),
    PRINT("print"),
    TYPE("int|string|boolean"),
    BOOLOP("==|!="),
    BOOLVAL("false|true"),
    INTOP("[+]"),
    CHAR("[a-z]"),
    DIGIT("[0-9]"),
    WHITESPACE("[ \t\f\r\n]+"),
    LPAREN("[(]"),
    RPAREN("[)]"),
    OTHER(".");
    public final String pattern;

    private TokenType(final String pattern) {
        this.pattern = pattern;
    }
}

public static class Token {
    public TokenType type;
    public String data;

    public Token(final TokenType type, final String data) {
        this.type = type;
        this.data = data;
    }
}

public static ArrayList<Token> lex(final String input) {
    // The tokens to return
    final ArrayList<Token> tokens = new ArrayList<Token>();

    // allows us to work with a mutable string
    final StringBuffer tokenPatternsBuffer = new StringBuffer();

    for (final TokenType tokenType : TokenType.values()) {
        tokenPatternsBuffer.append(String.format("|(?<%s>%s)",
                tokenType.name(), tokenType.pattern));
    }
    final Pattern tokenPatterns = Pattern.compile(
            new String(tokenPatternsBuffer.substring(1)));

    // Begin matching tokens
    String other = "";
    final Matcher matcher = tokenPatterns.matcher(input);
    while (matcher.find()) {
        if (matcher.group(TokenType.DIGIT.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.DIGIT,
                    matcher.group(TokenType.DIGIT.name())));
            continue;
        } else if (matcher.group(TokenType.IF.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.IF,
                    matcher.group(TokenType.IF.name())));
            continue;
        } else if (matcher.group(TokenType.WHILE.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.WHILE,
                    matcher.group(TokenType.WHILE.name())));
            continue;
        } else if (matcher.group(TokenType.TYPE.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.TYPE,
                    matcher.group(TokenType.TYPE.name())));
            continue;
        } else if (matcher.group(TokenType.PRINT.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.PRINT,
                    matcher.group(TokenType.PRINT.name())));
            continue;
        } else if (matcher.group(TokenType.BOOLOP.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.BOOLOP,
                    matcher.group(TokenType.BOOLOP.name())));
            continue;
        } else if (matcher.group(TokenType.BOOLVAL.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.BOOLVAL,
                    matcher.group(TokenType.BOOLVAL.name())));
            continue;
        } else if (matcher.group(TokenType.INTOP.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.INTOP,
                    matcher.group(TokenType.INTOP.name())));
            continue;
        } else if (matcher.group(TokenType.CHAR.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.CHAR,
                    matcher.group(TokenType.CHAR.name())));
            continue;
        } else if (matcher.group(TokenType.LPAREN.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.LPAREN,
                    matcher.group(TokenType.LPAREN.name())));
            continue;
        } else if (matcher.group(TokenType.RPAREN.name()) != null) {
            other = unknow(tokens, other);
            tokens.add(new Token(TokenType.RPAREN,
                    matcher.group(TokenType.RPAREN.name())));
            continue;
        } else if (matcher.group(TokenType.WHITESPACE.name()) != null) {
            continue;
        } else if (matcher.group(TokenType.OTHER.name()) != null) {
            other += matcher.group(TokenType.OTHER.name());
            continue;
        }
    }
    other = unknow(tokens, other);
    return tokens;
}

private static String unknow(final ArrayList<Token> tokens, final String _unknow) {
    if (!_unknow.isEmpty()) {
        tokens.add(new Token(TokenType.OTHER,_unknow));
    }
    return "";
}

public static void main(final String[] args) {
    final String input = "if\nprint\nta!?!?!taelse?§.?toto";
    // Create tokens and print them
    final ArrayList<Token> tokens = lex(input);
    for (final Token token : tokens)
        System.out.println("DEBUG Lexer - " + token.type + " [ "
                + token.data + " ] " + "found at " + "linenumber");
}

}

如何识别与模式不匹配的数据

1 个答案: