error[E0507]: Cannot move out of borrowed content

时间:2017-05-16 09:24:14

标签: rust lexer

I'm trying to make a lexer in Rust while being relatively new to it but with a background in C/C++. I'm having problems with how Rust allocates memory in the following code, which generates the error "Cannot move out of borrowed content". I've read cargo --explain E0507 which details possible solutions, but I'm struggling to grasp the underlying differences between how Rust and C/C++ manage memory. In essence, I want to understand how to manage dynamic memory in Rust (or a better way to achieve what I'm doing).

The error is:

error[E0507]: cannot move out of borrowed content
  --> <anon>:65:16
   |
65 |         return self.read_tok.unwrap();
   |                ^^^^ cannot move out of borrowed content

error[E0507]: cannot move out of borrowed content
  --> <anon>:73:16
   |
73 |         return self.peek_tok.unwrap();
   |                ^^^^ cannot move out of borrowed content

error: aborting due to 2 previous errors

The code is:

use std::fmt;

#[derive(Debug, PartialEq)]
pub enum TokenType {
    EndOfFile,
    Illegal
}

pub struct Token {
    token_type: TokenType,
    value: String
}

impl Token {
    pub fn new(token_type: TokenType, value: String) -> Token {
        return Token {
            token_type: token_type,
            value: value
        };
    }

    pub fn is_token_type(&self, token_type: TokenType) -> bool {
        return self.token_type == token_type;
    }
}

impl fmt::Debug for Token {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}[{}]", self.token_type, self.value)
    }
}

pub struct Lexer {
    input: String,
    read_pos: usize,
    peek_pos: usize,
    ch: char,
    read_tok: Option<Token>,
    peek_tok: Option<Token>
}

const EOF: char = 0 as char;

impl Lexer {
    pub fn new(input: &str) -> Lexer {
        return Lexer {
            input: input.to_string(),
            read_pos: 0,
            peek_pos: 1,
            ch: EOF,
            read_tok: None,
            peek_tok: None
        };
    }

    pub fn next_token(&mut self) -> Token {
        if self.peek_tok.is_none() {
            self.read_tok = Some(self.get_next_token());
        } else {
            self.read_tok = self.peek_tok.take();
        }

        return self.read_tok.unwrap();
    }

    pub fn peek_token(&mut self) -> Token {
        if self.peek_tok.is_none() {
            self.peek_tok = Some(self.get_next_token());
        }

        return self.peek_tok.unwrap();
    }

    fn get_next_token(&mut self) -> Token {
        let ch = self.next_char();
        let tok: Token;

        match ch {
            EOF => { tok = Token::new(TokenType::EndOfFile, "".to_string()); }
            _   => { tok = Token::new(TokenType::Illegal, ch.to_string()); }
        }

        return tok;
    }

    fn next_char(&mut self) -> char {
        if self.peek_pos >= self.input.len() {
            self.ch = EOF;
        } else {
            self.ch = self.input.chars().nth(self.peek_pos).unwrap();
        }

        self.read_pos = self.peek_pos;
        self.peek_pos += 1;

        return self.ch;
    }
}


fn main() {
    let input = "let x = 5;";
    let mut l = Lexer::new(input);

    loop {
        let t = l.next_token();
        println!("{:?}", t);

        if t.is_token_type(TokenType::EndOfFile) {
            break;
        }
    }
}

Rust playground link: https://play.rust-lang.org/?gist=bc85fafa35a5cbbd5ac4066aef9e333c&version=stable&backtrace=0https://play.rust-lang.org/?gist=21cba64f53488ee0a9389c0191c47134&version=stable&backtrace=0

I've managed to translate a working implementation in C++ which might give some more info on what I'm trying to achieve:

#include <string>
#include <iostream>

enum TokenType {
    ENDOFFILE,
    ILLEGAL
};

class Token {
private:
    enum TokenType token_type;
    std::string value;

public:
    Token(enum TokenType token_type, std::string value)
    {
        this->token_type = token_type;
        this->value = value;
    }

    bool is_token_type(enum TokenType token_type)
    {
        return this->token_type == token_type;
    }

    std::string to_string()
    {
        std::string tok;

        switch (this->token_type) {
        case ENDOFFILE:
            tok = "EndOfFile";
            break;
        case ILLEGAL:
            tok = "Illegal[" + this->value + "]";
            break;
        }

        return tok;
    }
};

class Lexer {
private:
    std::string input;
    int read_pos;
    int peek_pos;
    char ch;
    Token *read_tok;
    Token *peek_tok;

    Token *get_next_token() {
        char c = this->next_char();
        std::string c_str;
        Token *t;

        c_str.push_back(c);

        switch (c) {
        case 0:
            t = new Token(ENDOFFILE, "");
            break;
        default:
            t = new Token(ILLEGAL, c_str);
        }

        return t;
    }

    char next_char()
    {
        if (this->peek_pos >= this->input.length()) {
            this->ch = 0;
        } else {
            this->ch = input.at(this->peek_pos);
        }

        this->read_pos = this->peek_pos;
        this->peek_pos += 1;

        return this->ch;
    }

public:
    Lexer (std::string input)
    {
        this->input = input;
        this->read_pos = -1;
        this->peek_pos = 0;
        this->ch = 0;
        this->read_tok = NULL;
        this->peek_tok = NULL;
    }

    Token *next_token()
    {
        if (this->read_tok != NULL) {
            delete this->read_tok;
        }

        if (this->peek_tok == NULL) {
            this->read_tok = this->get_next_token();
        } else {
            this->read_tok = this->peek_tok;
            this->peek_tok = NULL;
        }

        return this->read_tok;
    }

    Token *peek_token()
    {
        if (this->peek_tok == NULL) {
            this->peek_tok = this->get_next_token();
        }

        return this->peek_tok;
    }
};

int main(int argc, char **argv)
{
    std::string input = "let x = 5;";
    Lexer l = Lexer(input);

    while (1) {
        Token *t = l.next_token();
        std::cout << t->to_string() << std::endl;

        if (t->is_token_type(ENDOFFILE)) {
            break;
        }
    }

    return 0;
}

1 个答案:

答案 0 :(得分:4)

你非常接近正确,但你的代码有两个问题。

首先,正如编译器告诉您的那样,禁止以下内容:

self.read_tok = self.peek_tok;
self.peek_tok = None;

第一行尝试将Option<Token>对象移出self.peek_tok。在Rust中,对象可以移出变量,但不能移出结构字段或切片下标。这是因为编译器可以检查移动后是否未使用该变量,以及安排不调用其析构函数。对于存储在结构字段或内部切片中的对象,这是不可能的,至少在没有为每个结构或容器添加开销的情况下都是如此。

只要将对象存储在支持移动的中间容器中,就可以将对象移出结构体。幸运的是,Option就是这样一个容器,它的take()方法就是为了这个目的而设计的:

self.read_tok = self.peek_tok.take()

Option::take()从选项中移动对象,将其替换为None,然后返回该对象。

其次,一旦修复了上述问题,编译器就会在returnnext_token的{​​{1}}语句中抱怨“移出借来的内容”,因为他们试图将对象移出peek_token的。{在这里,您可以选择克隆Option,或使用Token将其移出选项,如上所述。克隆方法需要将Option::take()添加到#[derive(Clone)]TokenType,并将返回值更改为:

Token

通过这些更改,示例将进行编译,但仍会将输入标记为非法。