Skip to main content

module lang::python::Parse

rascal-0.40.17
python-air-0.0.8

Uses the Python AST library to parse Python code and then maps it to a Rascal data-type.

Usage​

import lang::python::Parse;

Source code​

http://github.com/cwi-swat/python-air/blob/main/src/lang/python/Parse.rsc

Dependencies​

extend lang::python::AST;
import util::ShellExec;
import util::SystemAPI;
import util::FileSystem;
import util::Benchmark;
import lang::json::IO;
import IO;
import Type;
import ValueIO;
import String;
import List;

Description​

This module provides functions to produce abstract syntax trees for Python code. It supports expressions, statements and modules as top-level nodes.

The functionality provided here depends heavily on the shape of the ASTs provided by the Python AST library. This AST format is rather abstract and does not retain the order of appearance of syntactical constructs in a file. It does provide start and end line/column information for all expressions and statements.

This module is focused on enabling the most basic Python analysis in Rascal. It does not feature Python transformation features.

function installRequirements​

Installs the ast2json Python library using pip3.

void installRequirements()

function pythonPath​

Retrieves the search path for Python files using the sys.path constant.

list[loc] pythonPath()

function parsePythonExpression​

Parses a python expression producing an AST of type Expression.

Expression parsePythonExpression(str input, loc src)

Expression parsePythonExpression(str input)

function parsePythonStatement​

Parses a python statement producing an AST of type Expression.

Statement parsePythonStatement(str input, loc src)

Statement parsePythonStatement(str input)

function parsePythonModule​

Parses a python module producing an AST of type Module.

Module parsePythonModule(str input, loc src)

function parsePythonModule​

Parses a python module producing an AST of type Module.

Module parsePythonModule(loc src)

function importAST​

Wraps the python ast library as an external system process.

node importAST(str input)

This function starts the python3 interpreter to have access to the ast.parse function from Python's standard library. The output of the parser is mapped to JSON and then imported into a rascal data-structure that can be converted later.

This function should not be used by clients, but it can be called for debugging purposes since it reflects an intermediate stage.

function convertModule​

Module convertModule("object"(_type="Module", body=list[node] body, type_ignores=list[node] type_ignores), loc src)

Module convertModule("object"(_type="Expression", expr=node body), loc src)

Module convertModule("object"(_type="Interactive", body=list[node] body), loc src)

Module convertModule("object"(_type="FunctionType", argtypes=list[node] argtypes, expr=node returns), loc src)

function convertStat​

Statement convertStat(node obj:"object"(_type=str typ), loc src)

Statement convertStat("Module", node obj, loc src)

Statement convertStat("Expr", node obj, loc src)

Statement convertStat("FunctionDef",
    node obj:"object"(
        name=str name,
        args=node formals,
        body=list[node] body
    ),
    loc src)

Statement convertStat("AsyncFunctionDef",
    node obj:"object"(
        name=str name,
        args=node formals,
        body=list[node] body
    ),
    loc src)

Statement convertStat("ClassDef",
    node obj:"object"(
        name=str name,
        bases=list[node] bases,
        keywords=list[node] keywords,
        body=list[node] body
    ),
    loc src)

Statement convertStat("Return", node obj, loc src)

Statement convertStat("Delete", "object"(targets=list[node] targets), loc src)

Statement convertStat("Assign", node obj:"object"(targets=list[node] targets, \value=node \val), loc src)

Statement convertStat("AnnAssign", node obj:"object"(target=node target, annotation=node annotation), loc src)

Statement convertStat("AugAssign", "object"(op=node op, target=node target, \value=node v), loc src)

Statement convertStat("For", 
    node obj:"object"(
        target=node target,
        iter=node iter,
        body=list[node] body,
        orelse=list[node] orelse
    ),
    loc src
    )

Statement convertStat("AsyncFor", 
    node obj:"object"(
        target=node target,
        iter=node iter,
        body=list[node] body,
        orelse=list[node] orelse
    ),
    loc src
    )

Statement convertStat("While", 
    node obj:"object"(
        \test=node \test,
        body=list[node] body,
        orelse=list[node] orelse
    ),
    loc src
    )

Statement convertStat("If", 
    node obj:"object"(
        \test=node \test,
        body=list[node] body,
        orelse=list[node] orelse
    ),
    loc src
    )

Statement convertStat("With", 
    node obj:"object"(
        items=list[node] items,
        body=list[node] body
    ),
    loc src
    )

Statement convertStat("AsyncWith", 
    node obj:"object"(
        items=list[node] items,
        body=list[node] body
    ),
    loc src
    )

Statement convertStat("Raise", node obj, loc src)

Statement convertStat("Try", 
    node obj:"object"(
        body=list[node] body,
        handlers=list[node] handlers,
        orelse=list[node] orelse
    ),
    loc src)

Statement convertStat("Assert", node obj:"object"(\test=node t), loc src)

Statement convertStat("Import", "object"(names=list[node] aliases), loc src)

Statement convertStat("ImportFrom", node obj:"object"(names=list[node] aliases), loc src)

Statement convertStat("Global", "object"(names=list[str] names), loc src)

Statement convertStat("Nonlocal", "object"(names=list[str] names), loc src)

Statement convertStat("Pass", _, loc src)

Statement convertStat("Break", _, loc src)

Statement convertStat("Continue", _, loc src)

function convertExp​

Expression convertExp(node obj:"object"(_type=str typ), loc src)

default Expression convertExp(value v, loc src)

Expression convertExp("Module", node obj, loc src)

Expression convertExp("Expr", node obj, loc src)

Expression convertExp("Expression", node obj, loc src)

Expression convertExp("Attribute", "object"(\value=node v, attr=str a, ctx=node ctx), loc src)

Expression convertExp("Subscript", "object"(\value=node v, slice=node slice, ctx=node ctx), loc src)

Expression convertExp("Starred", "object"(\value=node v, ctx=node ctx), loc src)

Expression convertExp("Name", "object"(\id=str i, ctx=node ctx), loc src)

Expression convertExp("Tuple", "object"(elts=list[node] elts, ctx=node ctx), loc src)

Expression convertExp("Lambda", "object"(args=node args, body=node body), loc src)

Expression convertExp("Slice", node obj, loc src)

Expression convertExp("JoinedStr", "object"(values=list[node] values), loc src)

Expression convertExp("BinOp", "object"(op=node op, \left=node lhs, \right=node rhs), loc src)

Expression convertExp("Compare", "object"(left=node first, ops=list[node] ops, comparators=list[node] comps), loc src)

Expression convertExp("UnaryOp", "object"(op=node op, operand=node arg), loc src)

Expression convertExp("Name", "object"(ctx=node c, id=str n), loc src)

Expression convertExp("Call", "object"(func=node f, args=list[node] as, keywords=list[node] kws), loc src)

Expression convertExp("FormattedValue", node obj:"object"(\value=node v), loc src)

Expression convertExp("NamedExpr", "object"(target=node t, \value=node v), loc src)

Expression convertExp("List", "object"(elts=list[node] elts, ctx=node ctx), loc src)

Expression convertExp("Set", "object"(elts=list[node] elts), loc src)

Expression convertExp("Dict",  "object"(keys=list[node] keys, values=list[node] values), loc src)

Expression convertExp("BoolOp", "object"(op="object"(_type="And"), values=list[node] vs), loc src)

Expression convertExp("BoolOp", "object"(op="object"(_type="Or"), values=list[node] vs), loc src)

Expression convertExp("IfExp", "object"(\test=node t, body=node b, orelse=node orelse), loc src)

Expression convertExp("ListComp", "object"(elt=node e, generators=list[node] gens), loc src)

Expression convertExp("SetComp", "object"(elt=node e, generators=list[node] gens), loc src)

Expression convertExp("GeneratorExp", "object"(elt=node e, generators=list[node] gens), loc src)

Expression convertExp("DictComp", "object"(key=node k, \value=node v, generators=list[node] gens), loc src)

Expression convertExp("Await", "object"(\value=node e), loc src)

Expression convertExp("Yield", node obj, loc src)

Expression convertExp("YieldFrom", "object"(\value=node v), loc src)

Expression convertExp("Constant", "object"(\value=num v), loc src)

Expression convertExp("Constant", "object"(\value=str s), loc src)

default Expression convertExp("Constant", "object"(), loc src)

function convertItem​

WithItem convertItem(node obj:"object"(context_expr=node c), loc src)

function convertAlias​

Alias convertAlias(node obj:"object"(name=str name))

function convertHandler​

ExceptHandler convertHandler(
    node obj:"object"(
        body=list[node] body
    ),
    loc src)

function convertCompOp​

CmpOp convertCompOp("Eq")

CmpOp convertCompOp("NotEq")

CmpOp convertCompOp("Lt")

CmpOp convertCompOp("LtE")

CmpOp convertCompOp("Gt")

CmpOp convertCompOp("GtE")

CmpOp convertCompOp("Is")

CmpOp convertCompOp("IsNot")

CmpOp convertCompOp("In")

CmpOp convertCompOp("NotIn")

function convertAssign​

Statement convertAssign("Add", Expression target, Expression \value)

Statement convertAssign("Sub", Expression target, Expression \value)

Statement convertAssign("Mult", Expression target, Expression \value)

Statement convertAssign("Matmult", Expression target, Expression \value)

Statement convertAssign("Div", Expression target, Expression \value)

Statement convertAssign("Mod", Expression target, Expression \value)

Statement convertAssign("Pow", Expression target, Expression \value)

Statement convertAssign("LShift", Expression target, Expression \value)

Statement convertAssign("RShift", Expression target, Expression \value)

Statement convertAssign("BitOr", Expression target, Expression \value)

Statement convertAssign("BitXor", Expression target, Expression \value)

Statement convertAssign("BitAnd", Expression target, Expression \value)

Statement convertAssign("FloorDiv", Expression target, Expression \value)

function convertConv​

Conversion convertConv(int i)

Conversion convertConv(115)

Conversion convertConv(114)

Conversion convertConv(97)

function convertKeyword​

Keyword convertKeyword(node obj:"object"(\value=node v), loc src)

function convertGenerator​

Comprehension convertGenerator(
    "object"(
        target=node target, 
        iter=node iter, 
        ifs=list[node] ifs, 
        is_async=int isAsync)
    , loc src)

function convertOp​

Expression convertOp("Add", Expression l, Expression r)

Expression convertOp("Sub", Expression l, Expression r)

Expression convertOp("Mult", Expression l, Expression r)

Expression convertOp("MatMult", Expression l, Expression r)

Expression convertOp("Mod", Expression l, Expression r)

Expression convertOp("Pow", Expression l, Expression r)

Expression convertOp("Div", Expression l, Expression r)

Expression convertOp("LShift", Expression l, Expression r)

Expression convertOp("RShift", Expression l, Expression r)

Expression convertOp("BitOr", Expression l, Expression r)

Expression convertOp("BitXor", Expression l, Expression r)

Expression convertOp("BitAnd", Expression l, Expression r)

Expression convertOp("FloorDiv", Expression l, Expression r)

Expression convertOp("Invert", Expression a)

Expression convertOp("Not", Expression a)

Expression convertOp("UAdd", Expression a)

Expression convertOp("USub", Expression a)

function convertCtx​

ExprContext convertCtx("object"(_type="Load"))

ExprContext convertCtx("object"(_type="Store"))

ExprContext convertCtx("object"(_type="Del"))

function convertArgs​

Arguments convertArgs(
    node obj:"object"(
        _type="arguments", 
        posonlyargs=list[node] posonlyargs,
        kwonlyargs=list[node] kwonlyargs,
        args=list[node] args
    ),
    loc src)

function convertArg​

Arg convertArg(
    node obj:"object"(
        arg=str a
    ),
    loc src)

function convertTypeIgnore​

TypeIgnore convertTypeIgnore("object"(_type="TypeIgnore", lineno=int l, \tag=str t))

function offset​

int offset(int line, int column)

function offsets​

map[int line, int offset] offsets(str input)

function \loc​

loc \loc(loc path, int startLine, int startCol, int endLine, int endCol)

function pythonParserCode​

str pythonParserCode()

function \int​

int \int(value v)

function nodes​

list[node] nodes(value v)

function main​

void main()