如何使用Dart分析器从源生成AST并使用AST?

时间:2016-02-21 10:11:47

标签: dart

AST

文档:https://www.dartdocs.org/documentation/analyzer_experimental/0.8.0/analyzer/parseCompilationUnit.html

一些聪明的程序员在野外的优秀样本:https://github.com/mythz/dart-samples/blob/master/frog/leg/scanner/keyword.dart

RecursiveASTVisitor api:https://www.dartdocs.org/documentation/analyzer_experimental/0.8.0/analyzer/RecursiveASTVisitor-class.html

现在示例代码已移至下面的wiki。

飞镖分析服务器或飞镖服务

IDE或编辑器所依赖的包用于检测错误。

git:https://github.com/dart-lang/dart-services

安装:

运行服务器:

  • cd dart-services
  • dart ./bin/services.dart --port 8082

客户端代码:

import "dart:convert";
import "dart:io";


const String url = "localhost";
const String path = r"api/dartservices/v1/analyze";
String src = """
main(){
 print('hi');
}
""";

void main() {
  analyse();
}

analyse() async {
  Map m = {'source': src};
  var client = new HttpClient();
  String data = JSON.encode(m);
  HttpClientRequest request = await client.post (
      url, 8082, path);
  request.headers.contentType = ContentType.JSON;
  request.write(data);
  HttpClientResponse r = await request.close();
  r.transform(UTF8.decoder).listen((d){
    print(d);
  });
  client.close(force: false);
}

服务器将/lib/src/common_server.dart中的类CommonServer作为api提供。发布到" api / dartservices / v1 / analyze"在课堂上调用下面的方法。

  @ApiMethod(
      method: 'POST',
      path: 'analyze',
      description:
          'Analyze the given Dart source code and return any resulting '
          'analysis errors or warnings.')
  Future<AnalysisResults> analyze(SourceRequest request) {
    return _analyze(request.source);
  }

使用Analyzer作为本地对象。

import 'src/analyzer.dart';
import 'src/api_classes.dart';
import 'dart:developer';
String src = """
main(){
  a.print('hi');
}
""";
main() async {
  String sdkPath = r'C:\tools\dart-sdk';
  var a = new Analyzer(sdkPath);
  var m = {'main.dart':src};
  debugger();
  AnalysisResults r = await a.analyzeMulti(m);
  print(r.issues);
  print(r.resolvedImports);
}

分析器相当复杂,并且在微任务中占据了一席之地,调试器没有多大帮助。

3 个答案:

答案 0 :(得分:3)

分析器代码仍在进行中。我想从早期的Java源代码自动生成的Dart代码的返工还没有完成。他们正致力于改进公共API,然后他们也将致力于更好的文档。

我自己没有仔细看看,我认为这些包也应该使用它

答案 1 :(得分:3)

维基

注意:此处发布的代码示例质量不佳 应该有更简洁有效的方法,虽然到目前为止我没有找到它们。

有关更多详细信息,请在github上搜索dart-sdk以获取某些函数/类名称 这些都很有帮助:
https://github.com/dart-lang/sdk/blob/dd0c42cb72a4c218e4b04890f0ef666c6f6c0eb6/pkg/analyzer/test/dart/ast/visitor_test.dart
https://github.com/dart-lang/sdk/blob/71e7ed86c060ff2d695777d2c68c20e25cfb8ef9/pkg/analysis_server/lib/src/services/correction/util.dart
https://github.com/dart-lang/sdk/blob/6715573c6eb5e5a3c2b8257fef02e88a7d707d9f/pkg/analyzer/lib/src/generated/incremental_resolver.dart

对于AstVisitor(有几个):https://www.dartdocs.org/documentation/analyzer_experimental/0.8.0/analyzer/RecursiveASTVisitor-class.html

正如GünterZöchbauer所提到的那样,它正在进行中并且记录不清,而如果您正在编写用于编译时修改代码的变换器,则必须知道。

示例代码

查看树

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';
import 'dart:io';
//code to parse
String src = r"""
import 'package:mistletoe/mistletoe.dart';
var o = new Object();
main(){
  f(o);
}
f(p){
  var o = p;
  o = o as Object;
  print(o);
}
""";

main() async {
//  var src = await new File('./lib/mistletoe.dart').readAsString();
  var ast = parseCompilationUnit(src
    ,parseFunctionBodies: true);
  var nodes = flatten_tree(ast);
  var types ={};
  for(var n in nodes){
    types[n.runtimeType] ??= [];
    types[n.runtimeType].add(n);
  }
  var data = [];
  for(var k in types.keys){
    data.add(k.toString());
    for(var e in types[k]){
      data.add('\t'+e.toString());
    }
  }
  data = data.join('\n');
  print(data);
//  await new File('./lib/node_samples.txt').writeAsString(data);
}

List flatten_tree(AstNode n,[int depth=9999999]){
  var que = [];
  que.add(n);
  var nodes = [];
  int nodes_count = que.length;
  int dep = 0;
  int c = 0;
  if(depth == 0) return [n];
  while(que.isNotEmpty){
    var node = que.removeAt(0);
    if(node is! AstNode) continue;
    for(var cn in node.childEntities){
      nodes.add(cn);
      que.add(cn);
    }
    //Keeping track of how deep in the tree
    ++c;
    if(c == nodes_count){
      ++ dep; // One layer done
      if(depth <= dep) return nodes;
      c = 0;
      nodes_count = que.length;
    }
  }
  return nodes;
}
show(node){
  print('Type: ${node.runtimeType}, body: $node');
}

编辑节点

d.on(o).hi = 'bye'd.on(o).set("hi", 'bye')

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/error.dart';
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/generated/scanner.dart';
String src = """
  Dynamism d = new Dynamism(expert:true);
main(){
  var o = new Object();
  d.on(o).hi = 'bye';
}
""";
main(){
  var ast = parseCompilationUnit(src,parseFunctionBodies: true);
  print('initial value: ');
  print(ast.toSource());
  var v = new Visitor();
  ast.visitChildren(v);
  print('After modification:');
  print(ast.toSource());
}
class Visitor extends RecursiveAstVisitor{
  @override
  visitAssignmentExpression(AssignmentExpression node){
    //filter
    var p = new RegExp(r'.*\.on\(\w\)');
    if(!p.hasMatch(node.toString())) return;

    //replace
    SimpleStringLiteral ssl =
    _create_SimpleStringLiteral(node);
    node.parent.accept(new NodeReplacer(node,ssl));
  }
}

SimpleStringLiteral _create_SimpleStringLiteral(AstNode node){
  String new_string = modify(node.toString());
  int line_num = node.offset;
  //holds the position and type
  StringToken st = new StringToken(
      TokenType.STRING,new_string,line_num);
  return new SimpleStringLiteral(st, new_string);
}
String modify(String s){
  List parts = s.split('=');
  var value = parts[1];
  List l = parts[0].split('.');
  String dynamism = l.sublist(0,l.length-1).join('.');
  String propertyName = l.last.trim();
  return '${dynamism}.set("${propertyName}",${value})';
}

更改关键字:var o = new Object();Object o = new Object()

注意:编辑A VariableDeclarationList比编辑其他节点更困难。

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';

//code to parse
String src = r"""
var d = new Dynamism(expert: true);
var o = new Object();
void main() {
  var e = new Object();
}
""";
main(){
  var result = var_to_object(src);
  print(result);
}
String var_to_object(String s){
  var ast = parseCompilationUnit(
      s,parseFunctionBodies: true);

// list imports
//  var directives = ast.directives;
//  print(directives);

  var v = new Visitor();
  ast.visitChildren(v);
  return ast.toSource();
}
class Visitor extends RecursiveAstVisitor{
  _is_type_Object(n){
    for( var c in n.childEntities){
      if(c is ConstructorName){
        if(c.toString() == 'Object')
          return true;
      }
      if(c is VariableDeclarationList)
        return _is_type_Object(c);
      if(c is VariableDeclaration)
        return _is_type_Object(c);
      if(c is InstanceCreationExpression)
        return _is_type_Object(c);
    }
    return false;
  }
  @override
  visitVariableDeclarationList(VariableDeclarationList n){
    //if n does not contain new Dynamism(expert:true) return;
    if(!_is_type_Object(n)) return;

//    Some note on the Keyword class
//     pseudo keywords are keywords that can be used as identifiers.
//     e.g. 
//     const Keyword("await", isPseudo: true),
//     const Keyword("yield", isPseudo: true)];

// syntax arguments samples
//    static const Keyword LIBRARY = const Keyword._('LIBRARY', "library", true);
//    static const Keyword NEW = const Keyword._('NEW', "new");
//    static const Keyword NULL = const Keyword._('NULL', "null");
//    static const Keyword OPERATOR = const Keyword._('OPERATOR', "operator", true);

    // name and syntax seems to have been switched around
    var kw = const Keyword('class','Object', false);
    var kt = new KeywordToken(kw,n.keyword.offset);
    var ndl = new VariableDeclarationList(
        n.documentationComment,[],kt,
        n.type,n.variables);
    //changing var d to Object d
    n.parent.accept(new NodeReplacer(n,ndl));
  }
}

查找给定变量的最近声明/赋值/ FormalParameter / TypeName

注意:代码可能仍然包含错误。我应该重新组织代码,但现在我的时间有点短暂,并且在我原型时离开。

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';


String src = r"""
final String a = 'a';
class A{
  static Dynamism d = new Dynamism(expert:true);
  A(a){
    a = 'changed';
    print(a);
    var o = new Object();
    o = a as String;
    print(o);
  }
}""";

main() async{
  var ast = parseCompilationUnit(
      src,parseFunctionBodies: true);
  //Get all SimpleIdentifier nodes in ast
  var simple_nodes = new List<AstNode>();
  for(var n in flatten_tree(ast)){
    if(n is SimpleIdentifier){
      simple_nodes.add(n);
    }
  }
  for(SimpleIdentifier n in simple_nodes){
    var a = guess_effective_assignment_of(n);
    var d = get_declaration_of(n);
    var f = get_formal_parameter_of(n);
    if(a == null && d == null && f == null)
      continue;
    print('------------');
    print('For $n at: ${n.offset} in the node:'
        '${n.parent.parent}');
    print('The effective definition is:');
    var definitions = guess_effective_definition_of(n);
    var first_candidate = definitions.first;
    var second_candidate = definitions.second;
    var third_candidate = definitions.third;
    print('\t`${first_candidate}` of type '
        '${first_candidate.runtimeType}');
    print('\tSecond candidate is `${second_candidate}`');
    print('\tThird candidate is `${third_candidate}`');
    print('Where the enclosing block is:');
    print('\t${get_surrounding_block(n)}');
    var type = get_type_name(n);
    print('type or class name:$type');
  }

}
AstNode get_surrounding_block(node){
  if(node == null) return null;
  while(true){
    if(node.parent == null) return null;
    node = node.parent;
    if(is_scope(node))
      return node;

  }
}
String get_type_name(SimpleIdentifier node){
  Definitions definitions =
    guess_effective_definition_of(node);
  var g = definitions.first;
  //Takes only SimpleFormalParameter or Declaration
  TypeName _extract_TypeName_from(d){
    if(d == null) return null;
    for(var t in d.childEntities)
      if(t is TypeName) return t;
  }
  var name;
  if(g is VariableDeclaration){
    name = _extract_TypeName_from(g);
    if(name == null || name == 'var'){
      name = extract_constructor_name_or_rvalue(g);
      //Not supporting MethodInvocation
      if(name is MethodInvocation)
        return null;

      //rvalue is a variable. calling self.
      if(name is SimpleIdentifier)
        return get_type_name(name);
    }
  }
  if(g is AssignmentExpression){
    // checking FormalParameter or
    // VariableDeclaration for type
    var closer = definitions.second;
    var type_name = _extract_TypeName_from(closer);
    if(type_name != 'var' && type_name != null)
      return type_name.toString();
    //TypeName is var

    for(var e in g.childEntities){
      if(e is AsExpression){
        return e.childEntities.last.toString();
      }
    }
    name = extract_constructor_name_or_rvalue(g);
    //Not supporting MethodInvocation
    if(name is MethodInvocation)
      return null;

    //rvalue is a variable. calling self.
    if(name is SimpleIdentifier)
      return get_type_name(name);
    //A literal is handled later
  }
  if(g is FormalParameter){
    if(g.childEntities.length>1){
      name = _extract_TypeName_from(g);
    }else{
      name = 'var';
    }
  }

  if(name is Literal){
    name = name.runtimeType.toString();
    name = name.replaceAll('Simple','')
        .replaceAll('Literal','');
  }else{
    name = name.toString();
  }
  return name;
}


///finds the variable declaration for the given node
///Takes SimpleIdentifier representing a variable
List get_declaration_of(var n){
  // Check if n is part of VariableDeclaration
  // and is the variable being defined.
  var r = new List(2);
  var original = n;
  while(n.parent !=null){
    if(n is VariableDeclaration)
      if(n.childEntities.first == original){
        r[0]=n;r[1]=0;
        return r;
      }else{
        //n is not the variable being defined
        break;
      }
    n = n.parent;
  }
  n = original;

  // The variable's identifier is defined up the lines
  // or in outer scope.

  //searching local scope
  var block = get_surrounding_block(n);
  var declarations = extract_scope_wide_declarations(block);
  for(var d in declarations) {
    if(d.offset > n.offset) break;
    var v = d?.childEntities?.first;
    if(v == original){
      r[0]=d;r[1]=0;
      return r;}
  }

  //searching the outer scopes
  block = get_surrounding_block(block);
  declarations.clear();
  int count = -1;
  while(block != null){
    var decls = extract_scope_wide_declarations(block);
    for(var d in decls) {
      var v = d.childEntities.first;
      if(v.toString() == original.toString()){
        r[0]=d;r[1]=count;
        return r;
      }
    }
    block = get_surrounding_block(block);
    --count;
  }
  return null;//could not find the declaration
}
///Fetch declarations that have effect in
///all the children blocks of the given node.
///Does not cover FormalParameterList.
///Does not enter a node that is a block.
List extract_scope_wide_declarations(AstNode node){
  var nodes = [];
  var declarations = [];
  nodes.addAll(node.childEntities);

  //pushing more nodes and skipping blocks
  while(nodes.isNotEmpty){
    var e = nodes.removeAt(0);
    if(e is! AstNode) continue;
    if(e is VariableDeclaration){
      declarations.add(e);
      continue;
    }
    if(!is_scope(e) && e is AstNode){
        nodes.addAll(e.childEntities);
    }
  }
  return declarations;
}
///Returns true if the given node has its
///scope.
///Returns also true for:
/// MethodDeclaration
/// FunctionDeclaration
/// CompilationUnit
/// ClassDeclaration
///
///as they have a scope.
bool is_scope(node){
    if(node is Block||
        node is CompilationUnit ||
        node is Block||
        //below are for extracting arguments
        node is MethodDeclaration ||
        node is FunctionDeclaration ||
        node is ConstructorDeclaration||
        //FieldDeclarations
        node is ClassDeclaration ||
        //Block should cover these but jus in case
        node is IfStatement ||
        node is WhileStatement||
        node is DoStatement ||
        node is TryStatement ||
        node is SwitchStatement) return true;
  return false;
}

/// Takes a VariableDeclaration node.
/// Returns a ConstructorName node or
/// rvalue; includes MethodInvocation.
/// todo write test
extract_constructor_name_or_rvalue(n){
  var nodes = flatten_tree(n,1);
  for(var node in nodes){
    if(node is Literal) return node;
    if(node is TypeName){
      return node;
    }
    if(node is InstanceCreationExpression)
      for(var e in node.childEntities)
        if(e is ConstructorName) return e;
    if(node is MethodInvocation){
      return node;
    }
  }
  //rvalue is a variable
  return nodes.last;
}


/// Returns the closest definition of n.
/// sub blocks are ignored.
///
///  A guess because it would fail
/// if a conditional modifies the value
/// of the variable in runtime.
///
/// e.g.
///
///     var a = 'hi';
///     if(user_input){
///       a = new Object();
///     }
///     //a is an Object not String
///
/// also does not look into constructor.
///
/// todo write test
Definitions guess_effective_definition_of(n){
  List al = guess_effective_assignment_of(n);
  List dl = get_declaration_of(n);
  List fl = get_formal_parameter_of(n);

  if(al == null && dl == null && fl == null )
    return null;

  if(al == null && fl == null && dl != null)
    return new Definitions()
      ..first = dl[0]
      ..first_relative_depth=dl[1]
      ..length = 1;

  if(dl == null && fl == null && al != null)
    return new Definitions()
      ..first = al[0]
      ..first_relative_depth = al[1]
      ..length = 1;

  if(fl != null && al == null && dl == null)
    return new Definitions()
      ..first = fl[0]
      ..first_relative_depth = fl[1]
      ..length = 1;;

  Definitions r = new Definitions();
  // Guessing which declaration or
  // definition takes precedence.
  var l = []..add(dl)..add(al)..add(fl);
  l.removeWhere((e)=>e==null);

  // 0 means local scope.
  // -1 means one scope up.
  //Comparator returns negative if a comes before b,
  //0 if equal, positive if a comes after b.
  l.sort((a,b)=> b[1] - a[1]);
  // if AssignmentExpression and
  // VariableDeclaration are
  // in the same scope,
  // AssignmentExpression always
  // takes precedence.
  l.sort((a,b)=>
    a == al && b == dl && al[1] == dl[1] ?
      -1:0
  );
  //maybe I should just return a list?
  try {
    r.length = l.length;
    r.first = l[0][0];
    r.first_relative_depth = l[0][1];
    r.second = l[1][0];
    r.second_relative_depth = l[1][1];
    r.third = l[2][0];
    r.third_relative_depth = l[2][1];
  }on RangeError catch(e){
    // do nothing
  }
  return r;
}
/// Searches for an AssignmentExpression
/// that is most likely to define the value
/// of the variable denoted by the identifier
/// in n.
///
///  A guess because it would fail
/// if a conditional modifies the value
/// of the variable on runtime.
///
/// e.g.
///
///     var a = 'hi';
///     if(user_input){
///       a = new Object();
///     }
///     //a is an Object not String
///     //but this function does not
///     //know that.
///
guess_effective_assignment_of(SimpleIdentifier n){
  //checks if n is part of a variable declaration
  //Code is mostly duplicate of get_declaration_of
  // AssignmentExpression does not contain
  // VariableDeclaration or vice versa.
  var r = new List(2);
  var node = n;
  while(node.parent !=null){
    if(node is AssignmentExpression)
      if(node.childEntities.first == n){
        r[0]=node;r[1]=0;
        return r;
      }else{
        //n is not the variable being defined
        break;
      }
    node = node.parent;
  }

  //Search local scope
  var b = get_surrounding_block(n);
  AstNode closest;
  for(var node in extract_assignments_to_n_from(n, b)){
    if(n.offset > node.offset){
      closest = node;
    }else{break;}
  }
  if(closest != null){
    r[0]=closest;r[1]=0;
    return r;
  }
  //outer scopes
  b = get_surrounding_block(b);
  int count = -1;
  while(b != null){
    for(var a in extract_assignments_to_n_from(n,b)) {
      var v = a?.childEntities?.first;
      if(v.toString() == n.toString()){
        r[0]=a;r[1]=count;
        return r;
      }
    }
    b = get_surrounding_block(b);
    --count;
  }
  return null;
}

///Searches scopes upward for an
///AssignmentExpression for the
///variable denoted by the identifier
///in n.
///
/// Returns a list of AssignmentExpression.
/// todo test this
extract_assignments_to_n_from(
    SimpleIdentifier n,
    AstNode in_scope,
    [int search_depth=2]){
  var nodes = flatten_tree(in_scope,search_depth)
      .where((e)=>e is AssignmentExpression);
  var r = [];
  for(AstNode node in nodes){
    String i = node.childEntities.first.toString();
    if(i == n.toString()){
      r.add(node);
    }
  }
  return r;
}
///Takes any node.
///
///Searches scopes upward to find
///the closest FormalParameterList
///
/// Returns a list of
///   1.  FormalParameterList
///   2.  Its depth relative to n
///
/// Or null.
///
List get_nearest_formal_parameter_list(n){
  var r = new List(2);
  int count = 0;
  while(true){
    if(n is FunctionDeclaration ||
        n is MethodDeclaration ||
        n is ConstructorDeclaration)
      break;
    n = get_surrounding_block(n);
    --count;
    if(n == null) return n;
  }
  for(var e in n.childEntities){
    if(e is FormalParameterList){
      r[0] = e;r[1] = count;
      return r;
    }
  }
  //empty FormalParameterList
  return null;
}
///Searches up for a FormalParameterList.
///
///Returns a list of:
///
///  1. FormalParameter matching
///  the identifier of n when stringified
///  if such exists. Returns null
///  otherwise.
///
///   2.  The number of scopes moved up
///   from the scope n belongs to.
///
get_formal_parameter_of(n){
  var r = new List(2);

  //returns [FormalParameterList, int]
  var fl = get_nearest_formal_parameter_list(n);

  if( fl == null) return null;
  List names = extract_arg_names(fl[0]);
  for(var name in names){
    if(name.toString() == n.toString()){
      r[0]=name.parent;
      r[1]=fl[1];
      return r;
    }
  }
  return null;
}
///Takes:
///FunctionDeclaration
///FunctionExpression
///FormalParameterList
///
///Returns:
///Positional argument names :e.g. a and b in `f(a,b){return a+b;}`
///Named option names:e.g. your_name in `f({String your_name}){...}`
///
List<SimpleIdentifier> extract_arg_names(AstNode n){
  var fpl;
  if(n is! FormalParameterList){
    for(var e in flatten_tree(n)){
      if(e is FormalParameterList){
        fpl = e;
        break;
      }
    }
  }else if(n is FormalParameterList){
    fpl = n;
  }
  var r = [];
  for(var c in fpl.childEntities){
    if(c is! FormalParameter)
      continue;
    for(var cc in c.childEntities){
      //filtering String int etc
      if(cc is! TypeName) r.add(cc);
    }
  }
  return r;
}
List flatten_tree(AstNode n,[int depth=9999999]){
  var que = [];
  que.add(n);
  var nodes = [];
  int nodes_count = que.length;
  int dep = 0;
  int c = 0;
  if(depth == 0) return [n];
  while(que.isNotEmpty){
    var node = que.removeAt(0);
    if(node is! AstNode) continue;
    for(var cn in node.childEntities){
      nodes.add(cn);
      que.add(cn);
    }
    //Keeping track of how deep in the tree
    ++c;
    if(c == nodes_count){
      ++ dep; // One layer done
      if(depth <= dep) return nodes;
      c = 0;
      nodes_count = que.length;
    }
  }
  return nodes;
}
show(node){
  print('Type: ${node.runtimeType}, body: $node');
}
//todo test this: could not tokenize a node completely
class FlattenVisitor extends BreadthFirstVisitor{
  List _nodes;
  FlattenVisitor(this._nodes):super();
  @override
  visitNode(AstNode n){
    _nodes.add(n);
  }
}

/// Returned by guess_effective_definition_of
///
/// Confusing but a Definitions instance may
/// include `var a;`; a declaration but also
/// defining the type of a as dynamic.
///
/// Having moved upward to find the definition/
/// declaration results in a negative depth.
/// If definition/declaration is found locally,
/// depth is set to 0.
///
class Definitions{
  AstNode first;
  int first_relative_depth;
  AstNode second;
  int second_relative_depth;
  AstNode third;
  int third_relative_depth;
  int length;
}

答案 2 :(得分:0)

dart 2.11,来自测试代码

import 'package:analyzer/dart/analysis/utilities.dart';
import 'package:analyzer/dart/analysis/results.dart';
import 'package:test/test.dart';

void main() {
  test('hello_parser', () {
    String content = '''
void main() => print('Hello, world!')
''';
    ParseStringResult result = parseString(content: content, throwIfDiagnostics: false);
    // print(result.unit.toSource());
    expect(result.content, content);
    expect(result.errors, hasLength(1));
    expect(result.lineInfo, isNotNull);
    expect(result.unit.toString(), equals("void main() => print('Hello, world!');"));
  });
}