我正在升级之前使用的Ecmascript引擎
Quex 0.64.8到Quex 0.67.5。我有lexer和
运行，它似乎现在只能检测ANSI令牌，
而不是像以前那样的UTF-8令牌。
基本上，我所做的是提供--codec utf8标志
运行Quex并使用以下代码识别标识符：
PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ]
PATTERN_IDPART {PATTERN_IDSTART}|{PATTERN_DIGIT}
PATTERN_ID {PATTERN_IDSTART}{PATTERN_IDPART}*
我们的想法是指定我定义的所有允许令牌 这是不可接受的，不包括那些。新的词法分析器 检测到诸如“test1”或“safari”之类的标识符就好了，但似乎 遇到“日本语”和“Örjan”的问题。我也只使用utf-8 并且不要使用ICU或Iconv。
感觉好像我在这里误解了一些东西。任何帮助 解决这个问题将非常感激。
编辑：
知道我使用以下参数运行Quex可能很有用：
-i ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx
--analyzer-class ecmascript_lexer
--foreign-token-id-file ${BISON_ECMASCRIPT_PARSER_OUTPUT_HEADER}
--token-id-prefix TOK_
--template-compression
--codec utf8 //--encoding utf8 since Quex 0.67.5
--buffer-element-size 1
--buffer-element-type char
--odir ${CMAKE_CURRENT_BINARY_DIR}/generated
--language c++
--warning-on-outrun
编辑2：
自utf-8解析以来，我没有重新创建一个小例子 在这个例子中工作。因此我创建了一个独立的 相反，我的ecmascript引擎的lexer部分的版本， 希望能更容易看出出了什么问题。
我不再清楚我的问题是否真的与之相关 解析utf8令牌。很可能有些事情可能是错的 在我的.qx文件中......不管怎样，这里是独立版本 我的ecmascript lexer。
的CMakeLists.txt
cmake_minimum_required(VERSION 2.8)
project(ecmascript CXX)
if(MSVC)
add_definitions(-D_CRT_SECURE_NO_WARNINGS)
endif()
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
set(QUEX_NAMES "quex")
if(CMAKE_HOST_WIN32)
set(QUEX_NAMES "quex.bat" ${QUEX_NAMES})
else()
set(QUEX_NAMES "quex-exe.py" ${QUEX_NAMES})
endif()
find_program(QUEX NAMES ${QUEX_NAMES} REQUIRED
HINTS ENV QUEX_PATH DOC "Path to Quex's executable."
NO_DEFAULT_PATH)
find_path(QUEX_INCLUDE_DIR quex/core.py REQUIRED
HINTS ENV QUEX_PATH DOC "Path to Quex's include directory"
NO_DEFAULT_PATH)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/generated)
add_definitions(-DQUEX_OPTION_LINE_NUMBER_COUNTING
-DQUEX_OPTION_ASSERTS_DISABLED)
add_definitions(-DQUEX_SETTING_BUFFER_SIZE=1024) # Sätter bufferstorleken på lexern
set(ECMASCRIPT_LEXER ${CMAKE_CURRENT_BINARY_DIR}/generated/ecmascript_lexer)
add_custom_command(OUTPUT ${ECMASCRIPT_LEXER}
${ECMASCRIPT_LEXER}.cpp
${ECMASCRIPT_LEXER}-token
${ECMASCRIPT_LEXER}-configuration
${ECMASCRIPT_LEXER}-token_ids
COMMAND ${QUEX} -i ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx
--analyzer-class ecmascript_lexer # Namnet på lexern
--foreign-token-id-file ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript_yacc.hpp # token-id genereras av bison
--token-id-prefix TOK_ # Custom prefix för tokens (se ecmascript.y för detaljer)
--template-compression # Optimera lägesövergångar om möjligt
--encoding utf8 # Basera lexern på teckentabell UTF8
--buffer-element-size 1 # Använd en datatyp som är 1 byte stor
--buffer-element-type uint8_t
--odir ${CMAKE_CURRENT_BINARY_DIR}/generated # Berätta var alla genererade filer ska ligga
--language c++
DEPENDS ecmascript.qx VERBATIM
COMMENT "Generating ecmascript lexer..."
MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/ecmascript.qx) # Detta styr i vilken ordning Quex och Bison körs
include_directories(${QUEX_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}/generated)
set(es_lexer ${ECMASCRIPT_LEXER}
${ECMASCRIPT_LEXER}.cpp
_main.cpp)
set(es_generated ${es_lexer} ecmascript_yacc.hpp)
add_executable(es_lexer ${es_generated})
ecmascript.qx
header {
#include <quex/code_base/extra/accumulator/Accumulator>
#include "ecmascript_yacc.hpp"
#include <cstdlib>
#include <cstdio>
#define BACKSPACE '\x08'
#define TAB '\x09'
#define NEWLINE '\x0A'
#define VERTICALTAB '\x0B'
#define FORMFEED '\x0C'
#define CARRIAGERETURN '\x0D'
#define DOUBLEQUOTE '\x22'
#define SINGLEQUOTE '\x27'
#define DOUBLEBACKSLASH '\x5C'
#define NULLTERM '\x00'
}
footer {
#include <quex/code_base/extra/accumulator/Accumulator.i>
}
define {
PATTERN_NEWLINE [\n\r]
PATTERN_DIGIT [0-9]
PATTERN_NOZDIGIT [1-9]
PATTERN_DECINTLIT "0"|{PATTERN_NOZDIGIT}{PATTERN_DIGIT}*
PATTERN_EXPIND "e"|"E"
PATTERN_SIGNEDINT {PATTERN_DIGIT}+|"+"{PATTERN_DIGIT}+|"-"{PATTERN_DIGIT}+
PATTERN_EXPPART {PATTERN_EXPIND}{PATTERN_SIGNEDINT}
PATTERN_DECNUMBER {PATTERN_DECINTLIT}"."{PATTERN_DIGIT}*{PATTERN_EXPPART}?|"."{PATTERN_DIGIT}+{PATTERN_EXPPART}?|{PATTERN_DECINTLIT}{PATTERN_EXPPART}?
PATTERN_HEXDIGIT [0-9a-fA-F]
PATTERN_HEXNUMBER "0x"{PATTERN_HEXDIGIT}+|"0X"{PATTERN_HEXDIGIT}+
PATTERN_UNIESCSEQ \\u{PATTERN_HEXDIGIT}{4}
PATTERN_STRING "\""(\\"\""|[^"])*"\""
PATTERN_DOUBLE_QUOTE_STRING_DELIMITER "\""
PATTERN_SINGLE_QUOTE_STRING_DELIMITER "'"
PATTERN_SINGLELINE_COMMENT "//"[^\n\r]*
PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ]
PATTERN_IDPART {PATTERN_IDSTART}|{PATTERN_DIGIT}
PATTERN_ID {PATTERN_IDSTART}{PATTERN_IDPART}*
}
mode EOF : <inheritable: only> {
on_end_of_stream {
self_send(TOK_LINETERM);
self_send(TOK_TERMINATION);
}
}
mode RestrictedProduction : EOF
<skip: [ \t]>
{
{PATTERN_NEWLINE}{
self_send(';');
self << Program;
}
on_failure {
self.undo();
self << Program;
}
}
mode StringHelper : EOF
<inheritable: only>
{
on_entry {
self_send(TOK_QUOTE);
}
on_exit {
if(self.accumulator.text.begin != self.accumulator.text.end)
self_send(TOK_STRLITPART);
self_accumulator_flush(TOK_QUOTE);
}
{PATTERN_NEWLINE} => '\n';
"\\b" { self_accumulator_add_character(BACKSPACE); }
"\\t" { self_accumulator_add_character(TAB); }
"\\n" { self_accumulator_add_character(NEWLINE); }
"\\v" { self_accumulator_add_character(VERTICALTAB); }
"\\f" { self_accumulator_add_character(FORMFEED); }
"\\r" { self_accumulator_add_character(CARRIAGERETURN); }
"\\\"" { self_accumulator_add_character(DOUBLEQUOTE); }
"\\'" { self_accumulator_add_character(SINGLEQUOTE); }
"\\\\" { self_accumulator_add_character(DOUBLEBACKSLASH); }
"\\0" { self_accumulator_add_character(NULLTERM); }
"\\x"{PATTERN_HEXDIGIT}{2}
{
{
unsigned long ulResult = strtoul(reinterpret_cast<char*>(Lexeme+2),0,16);
uint8_t *const pBuffer = reinterpret_cast<uint8_t*>(&ulResult);
self_accumulator_add(pBuffer,pBuffer+2);
}
}
on_failure {
self_accumulator_add(Lexeme, LexemeEnd);
}
}
mode SingleQuoteString : StringHelper
{
{PATTERN_SINGLE_QUOTE_STRING_DELIMITER}
{
// Om vi hittade slutet på strängen så växlar vi tillbaka till Program-läget
self << Program;
}
}
mode DoubleQuoteString : StringHelper
{
{PATTERN_DOUBLE_QUOTE_STRING_DELIMITER}
{
// Om vi hittade slutet på strängen så växlar vi tillbaka till Program-läget
self << Program;
}
}
mode PrefixHelper : EOF
<skip: [ \t]> // Ignorera whitespace
{
on_entry {
self.seek_backward(3);
}
{PATTERN_NEWLINE}
{
if(self.iParaCount == 0)
self_send(';');
}
"++"
{
self_send(TOK_PLUSPLUS);
self << Program;
}
"--"
{
self_send(TOK_MINUSMINUS);
self << Program;
}
on_failure {
(void)Lexeme;
}
}
mode Operators : <inheritable: only>
{
"||" => TOK_OR;
"&&" => TOK_AND;
"++" { self << PrefixHelper; }
"--" { self << PrefixHelper; }
"===" => TOK_EQEQEQ;
"==" => TOK_EQEQ;
"!==" => TOK_NEQEQ;
"!=" => TOK_NEQ;
"*=" => TOK_MULTEQ;
"/=" => TOK_DIVEQ;
"%=" => TOK_MODEQ;
"+=" => TOK_PLUSEQ;
"\-=" => TOK_MINUSEQ;
">>>=" => TOK_GTGTGTEQ;
">>>" => TOK_GTGTGT;
"<<=" => TOK_LTLTEQ;
">>=" => TOK_GTGTEQ;
"<<" => TOK_LTLT;
">>" => TOK_GTGT;
"<=" => TOK_LTE;
">=" => TOK_GTE;
"&=" => TOK_AMPEQ;
"^=" => TOK_CIRCEQ;
"|=" => TOK_PIPEEQ;
['='] => '=';
['!'] => '!';
['('] { self_send('('); ++self.iParaCount; }
['+'] => '+';
['\-'] => '-';
['*'] => '*';
['/'] => '/';
['%'] => '%';
['<'] => '<';
['>'] => '>';
['\['] => '[';
['\]'] => ']';
['.'] => '.';
[','] => ',';
['?'] => '?';
[':'] => ':';
['~'] => '~';
['&'] => '&';
['^'] => '^';
['|'] => '|';
['{'] => '{';
[';'] => ';';
[')'] { self_send(')'); --self.iParaCount; }
['}'] { self_send(TOK_LINETERM); self_send('}'); }
}
mode Keywords : <inheritable: only>
{
function => TOK_FUNCTION;
return { self_send(TOK_RETURN); self << RestrictedProduction; }
var => TOK_VAR;
null => TOK_NULL;
true => TOK_TRUE;
false => TOK_FALSE;
instanceof => TOK_INSTANCEOF;
in => TOK_IN;
delete => TOK_DELETE;
void => TOK_VOID;
typeof => TOK_TYPEOF;
this => TOK_THIS;
if => TOK_IF;
else => TOK_ELSE;
with => TOK_WITH;
throw { self_send(TOK_THROW); self << RestrictedProduction; }
try => TOK_TRY;
catch => TOK_CATCH;
finally => TOK_FINALLY;
for => TOK_FOR;
break { self_send(TOK_BREAK); self << RestrictedProduction; }
continue { self_send(TOK_CONTINUE); self << RestrictedProduction; }
while => TOK_WHILE;
do => TOK_DO;
switch => TOK_SWITCH;
case => TOK_CASE;
default => TOK_DEFAULT;
new => TOK_NEW;
synchronized => TOK_SYNCHRONIZED;
}
mode Values : <inheritable: only>
{
{PATTERN_DECNUMBER} => TOK_DECLIT(Lexeme);
{PATTERN_HEXNUMBER} => TOK_HEXINTLIT(Lexeme);
{PATTERN_DOUBLE_QUOTE_STRING_DELIMITER} { self << DoubleQuoteString; }
{PATTERN_SINGLE_QUOTE_STRING_DELIMITER} { self << SingleQuoteString; }
}
mode Identifiers : <inheritable: only>
{
{PATTERN_ID} => TOK_ID(Lexeme);
}
mode Program : Keywords,
Identifiers,
Values,
Operators,
EOF
<skip: [ \t]>
<skip_range: "/*" "*/">
{
{PATTERN_NEWLINE}
{
if(self.iParaCount == 0)
self_send(TOK_LINETERM);
}
{PATTERN_SINGLELINE_COMMENT}
{}
}
body {
void push_token(const unsigned int uiToken)
{
self.uiLastToken = self.uiCurrentToken;
self.uiCurrentToken = uiToken;
}
bool use_auto_semi() const
{ return uiLastToken == TOK_LINETERM; }
unsigned int uiLastToken,
uiCurrentToken;
int iParaCount;
quex::Token* pLastID;
QUEX_NAME(Accumulator) accumulator;
}
constructor {
self.uiLastToken = 0;
self.uiCurrentToken = 0;
self.iParaCount = 0;
self.pLastID = 0;
if(!QUEX_NAME(Accumulator_construct)(&me->accumulator, me)) {
return false;
}
}
destructor {
QUEX_NAME(Accumulator_destruct)(&me->accumulator);
}
start = Program;
ecmascript_yacc.hpp
#ifndef YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED
# define YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED
/* Token type. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
enum yytokentype
{
TOK_TERMINATION = 0,
TOK_UNINITIALIZED = 1,
TOK_ID = 258,
TOK_NULL = 259,
TOK_TRUE = 260,
TOK_FALSE = 261,
TOK_DECLIT = 262,
TOK_HEXINTLIT = 263,
TOK_OR = 264,
TOK_AND = 265,
TOK_PLUSPLUS = 266,
TOK_MINUSMINUS = 267,
TOK_EQEQ = 268,
TOK_NEQ = 269,
TOK_EQEQEQ = 270,
TOK_NEQEQ = 271,
TOK_LTE = 272,
TOK_GTE = 273,
TOK_INSTANCEOF = 274,
TOK_IN = 275,
TOK_STRLITPART = 276,
TOK_QUOTE = 277,
TOK_VOID = 278,
TOK_TYPEOF = 279,
TOK_DELETE = 280,
TOK_THIS = 281,
TOK_LTLT = 282,
TOK_GTGT = 283,
TOK_GTGTGT = 284,
TOK_MULTEQ = 285,
TOK_DIVEQ = 286,
TOK_MODEQ = 287,
TOK_PLUSEQ = 288,
TOK_MINUSEQ = 289,
TOK_LTLTEQ = 290,
TOK_GTGTEQ = 291,
TOK_GTGTGTEQ = 292,
TOK_AMPEQ = 293,
TOK_CIRCEQ = 294,
TOK_PIPEEQ = 295,
TOK_IF = 296,
TOK_ELSE = 297,
TOK_RETURN = 298,
TOK_VAR = 299,
TOK_WITH = 300,
TOK_THROW = 301,
TOK_TRY = 302,
TOK_CATCH = 303,
TOK_FINALLY = 304,
TOK_FOR = 305,
TOK_BREAK = 306,
TOK_CONTINUE = 307,
TOK_WHILE = 308,
TOK_DO = 309,
TOK_SWITCH = 310,
TOK_CASE = 311,
TOK_DEFAULT = 312,
TOK_NEW = 313,
TOK_FUNCTION = 314,
TOK_SYNCHRONIZED = 315,
TOK_LINETERM = 316
};
#endif
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
typedef int YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
# define YYSTYPE_IS_DECLARED 1
#endif
#endif /* !YY_ECMASCRIPT_YY_C_USERS_PATRIKJ_WORK_GIT_ECMASCRIPT_BUILD_VC14_X64_GENERATED_ECMASCRIPT_YACC_HPP_INCLUDED */
_main.cpp
#include <iostream>
#include "ecmascript_lexer"
/****************************************************************************************/
void print_token(quex::Token* token)
{
std::cout << token->get_string() << std::endl;
}
/****************************************************************************************/
int main(int argc, char** argv)
{
quex::Token* token = 0;
quex::ecmascript_lexer qlex;
quex::ecmascript_lexer *lexer = quex::ecmascript_lexer::from_file_name("id_test.js", 0);
while(lexer->error_code == E_Error_None)
{
get_token:
lexer->receive(&token);
if(!token)
break;
print_token(token);
lexer->push_token(token->type_id());
if(token->type_id() == TOK_LINETERM)
goto get_token;
if(token->type_id() == TOK_ID)
lexer->pLastID = token;
if(token->type_id() == TOK_TERMINATION)
break;
}
delete lexer;
return 0;
}
id_test.js //用于测试词法分析器
test1 = safari;
myFunc()
function t(){}
if(test1 < 23)
return myFunc(45);
myFunc();
svenskaåäö();
var kalleö = 34;
var _us=kalleö;
_us = 678
日本語 = "Nihongo" // Japanska språkets namn
$myCar = _us
var new1 = kalleö ? t();
"kalleÖ, _us and $myCar should be ignored here"
الفصحى = "Arabiska"
/*
var new1 = kalleÖ ? t();
"kalleÖ, _us and $myCar should be ignored here"
*/
// var new1 = kalleÖ ? t();
대한민국 = 45;
대한민국X45 = "Value of: 대한민국" + 대한민국;
ärta="ärta + 2"
mix帝With대한민국 = "success?";
Örjan;
önes;
cake;
Россия;
РоссияX;
РоссияX
XРоссия;
XРоссия;
始皇帝 = "The First emperor"
始皇帝x2 = "The First emperor, twice?"
致以最诚挚的问候，
Patrik J
答案 0 :(得分：1)
我建议您特别依赖Unicode属性
ID_Start和ID_Continue，以使您的.qx文件包含
define {
ID_START \P{ID_Start}
ID_CONTINUE \P{ID_Continue}
ID {ID_START}{ID_CONTINUE}*
}
然后Quex对UCS数据库进行采样，您不必担心 特定的代码点。
此外，如果您只想支持子集，请使用intersection
切出所需的UCS范围，如下例所示：
...
ID_START [: intersection([\X900-\X970], \P{ID_Start}) :]
ID_CONTINUE [: intersection([\X900-\X970], \P{ID_Continue}) :]
...
PS，你的解决方案并非完全错误。给定一个文件example.qx：
define {
PATTERN_IDSTART [^0-9+\-<>*()\[\]?=&|~\\/\^%!{}\n\t\r"':;,. ]
PATTERN_IDPART {PATTERN_IDSTART}|[0-9]
PATTERN_ID {PATTERN_IDSTART}{PATTERN_IDPART}*
}
token { ID; WS; }
mode X {
{PATTERN_ID} => QUEX_TKN_ID(Lexeme);
[ \n\t] => QUEX_TKN_WS(Lexeme);
}
一些用户文件'example.c'：
#include <stdio.h>
#include "EasyLexer.h"
void
print_token(quex_Token* token_p)
{
const size_t BufferSize = 1024;
char buffer[1024];
printf("%s \n", QUEX_NAME_TOKEN(get_string)(token_p, buffer, BufferSize));
}
int
main(int argc, char** argv)
{
quex_Token* token_p = NULL;
quex_EasyLexer qlex;
quex_EasyLexer_from_file_name(&qlex, "example.txt", NULL);
while( qlex.error_code == E_Error_None ) {
quex_EasyLexer_receive(&qlex, &token_p);
if( ! token_p ) break;
print_token(token_p);
if( token_p->_id == QUEX_TKN_TERMINATION ) break;
}
quex_EasyLexer_destruct(&qlex);
return 0;
}
然后在命令行上执行：
> quex -i tmp.qx --encoding utf8 --language C -o EasyLexer
> gcc -I$QUEX_PATH example.c EasyLexer.c -o example
> ./example example.txt
递送
ID 'Örjan'
WS '\n'
ID '日本語'
WS '\n'
鉴于文件'example.txt'是UTF-8编码并包含
Örjan
日本語
我不知道该说些什么。我理解错了吗？