[WIP] Implement (possibly?) the entire Rust grammar.

build.rs

@@ -21,6 +21,22 @@ fn main() {
     // Start with the builtin rules for proc-macro grammars.
     let mut grammar = gll::proc_macro::builtin();
 
+    // HACK(eddyb) inject a custom builtin - this should be upstreamed to gll!
+    {
+        use gll::proc_macro::{FlatTokenPat, Pat};
+
+        grammar.define(
+            "LIFETIME",
+            gll::grammar::eat(Pat(vec![
+                FlatTokenPat::Punct {
+                    ch: Some('\''),
+                    joint: Some(true),
+                },
+                FlatTokenPat::Ident(None),
+            ])),
+        );
+    }
+
     // Add in each grammar fragment to the grammar.
     for fragment in fragments {
         let path = fragment.into_path();

grammar/abi.g

@@ -0,0 +1,32 @@
+// FIXME(eddyb) implement more specific literal support in `gll::proc_macro`
+Abi = LITERAL?;
+
+/*
+// HACK(eddyb) taken from `librustc_target/spec/abi.rs`
+Abi =
+    // Defaults to "C"
+    {} |
+
+    // Platform-specific ABIs
+    "\"cdecl\"" |
+    "\"stdcall\"" |
+    "\"fastcall\"" |
+    "\"vectorcall\"" |
+    "\"thiscall\"" |
+    "\"aapcs\"" |
+    "\"win64\"" |
+    "\"sysv64\"" |
+    "\"ptx-kernel\"" |
+    "\"msp430-interrupt\"" |
+    "\"x86-interrupt\"" |
+    "\"amdgpu-kernel\"" |
+
+    // Cross-platform ABIs
+    "\"Rust\"" |
+    "\"C\"" |
+    "\"system\"" |
+    "\"rust-intrinsic\"" |
+    "\"rust-call\"" |
+    "\"platform-intrinsic\"" |
+    "\"unadjusted\"";
+*/

grammar/attr.g

@@ -1,9 +1,7 @@
 OuterAttr = "#" attr:Attr;
-InnerAttr = "#!" attr:Attr;
+InnerAttr = "#" "!" attr:Attr;
 Attr = "[" path:Path input:AttrInput "]";
 AttrInput =
     {} |
-    "=" LITERAL |
-    "(" TOKEN_TREE* ")" |
-    "[" TOKEN_TREE* "]" |
-    "{" TOKEN_TREE* "}";
+    "=" TOKEN_TREE |
+    MacroInput;

grammar/expr.g

@@ -0,0 +1,116 @@
+Expr = attrs:OuterAttr* kind:ExprKind;
+ExprKind =
+    Literal:LITERAL |
+    Paren:{ "(" attrs:InnerAttr* expr:Expr ")" } |
+    Borrow:{ "&" mutable:"mut"? expr:Expr } |
+    Box:{ "box" expr:Expr } |
+    Unary:{ op:UnaryOp expr:Expr } |
+    Try:{ expr:Expr "?" } |
+    Binary:{ left:Expr op:BinaryOp right:Expr } |
+    Assign:{ left:Expr { "=" | op:BinaryAssignOp } right:Expr } |
+    Range:{ start:Expr? ".." end:Expr? } |
+    RangeInclusive:{ start:Expr? "..=" end:Expr } |
+    Type:{ expr:Expr ":" ty:Type } |
+    Cast:{ expr:Expr "as" ty:Type } |
+    Field:{ base:Expr "." field:FieldName } |
+    Index:{ base:Expr "[" index:Expr "]" } |
+    Array:{ "[" attrs:InnerAttr* exprs:Expr* % "," ","? "]" } |
+    Repeat:{ "[" attrs:InnerAttr* elem:Expr ";" count:Expr "]" } |
+    Tuple:{ "(" attrs:InnerAttr* exprs:Expr* % "," ","? ")" } |
+    Path:QPath |
+    Call:{ callee:Expr "(" args:Expr* % "," ","? ")" } |
+    MethodCall:{ receiver:Expr "." method:PathSegment "(" args:Expr* % "," ","? ")" } |
+    Struct:{ path:Path "{" attrs:InnerAttr* fields:StructExprFieldsAndBase "}" } |
+    Block:{ { label:Label ":" }? unsafety:"unsafe"? block:Block } |
+    AsyncBlock:{ "async" block:Block } |
+    TryBlock:{ "try" block:Block } |
+    Continue:{ "continue" label:Label? } |
+    Break:{ "break" label:Label? value:Expr? } |
+    Return:{ "return" value:Expr? } |
+    Yield:{ "yield" value:Expr? } |
+    If:If |
+    Match:{ "match" expr:Expr "{" attrs:InnerAttr* arms:MatchArm* "}" } |
+    Loop:{ { label:Label ":" }? "loop" body:Block } |
+    While:{ { label:Label ":" }? "while" cond:Cond body:Block } |
+    For:{ { label:Label ":" }? "for" pat:Pat "in" expr:Expr body:Block } |
+    Closure:{
+        generator_static:"static"? asyncness:"async"? by_val:"move"?
+        "|" args:ClosureArg* % "," ","? "|" { "->" ret_ty:Type }? body:Expr
+    } |
+    MacroCall:MacroCall;
+
+UnaryOp =
+    Not:"!" |
+    Neg:"-" |
+    Deref:"*";
+
+BinaryOp =
+    // Arithmetic & bitwise (these also have `BinaryAssignOp` forms)
+    Add:"+" |
+    Sub:"-" |
+    Mul:"*" |
+    Div:"/" |
+    Rem:"%" |
+    BitXor:"^" |
+    BitAnd:"&" |
+    BitOr:"|" |
+    Shl:"<<" |
+    Shr:">>" |
+
+    // Logic (short-circuiting)
+    LogicAnd:"&&" |
+    LogicOr:"||" |
+
+    // Comparisons
+    Eq:"==" |
+    Lt:"<" |
+    Le:"<=" |
+    Ne:"!=" |
+    Gt:">" |
+    Ge:">=";
+
+// FIXME(eddyb) figure out how to deduplicate this with `BinaryOp`
+// The problem is something like `BinaryOp "="` allows a space in between,
+// as the last token inside each `BinaryOp` case does not require being
+// joint, but each token before the `=` here does impose that requirement.
+BinaryAssignOp =
+    Add:"+=" |
+    Sub:"-=" |
+    Mul:"*=" |
+    Div:"/=" |
+    Rem:"%=" |
+    BitXor:"^=" |
+    BitAnd:"&=" |
+    BitOr:"|=" |
+    Shl:"<<=" |
+    Shr:">>=";
+
+FieldName =
+    Ident:IDENT |
+    // FIXME(eddyb) restrict this to only integer literals
+    Numeric:LITERAL;
+
+// FIXME(eddyb) find a way to express this `A* B?` pattern better
+StructExprFieldsAndBase =
+    Fields:{ fields:StructExprField* % "," ","? } |
+    Base:{ ".." base:Expr } |
+    FieldsAndBase:{ fields:StructExprField+ % "," "," ".." base:Expr };
+
+StructExprField = attrs:OuterAttr* kind:StructExprFieldKind;
+StructExprFieldKind =
+    Shorthand:IDENT |
+    Explicit:{ field:FieldName ":" expr:Expr };
+
+Label = LIFETIME;
+
+If = "if" cond:Cond then:Block { "else" else_expr:ElseExpr }?;
+Cond =
+    Bool:Expr |
+    Let:{ "let" pat:Pat "=" expr:Expr };
+ElseExpr =
+    Block:Block |
+    If:If;
+
+MatchArm = attrs:OuterAttr* "|"? pats:Pat+ % "|" { "if" guard:Expr }? "=>" body:Expr ","?;
+
+ClosureArg = pat:Pat { ":" ty:Type }?;

grammar/generics.g

@@ -0,0 +1,35 @@
+Generics = "<" params:GenericParam* % "," ","? ">";
+GenericParam = attrs:OuterAttr* kind:GenericParamKind;
+GenericParamKind =
+    Lifetime:{ name:LIFETIME { ":" bounds:LifetimeBound* % "+" "+"? }? } |
+    Type:{ name:IDENT { ":" bounds:TypeBound* % "+" "+"? }? { "=" default:Type }? };
+
+ForAllBinder = "for" generics:Generics;
+
+WhereClause = "where" bounds:WhereBound* % "," ","?;
+WhereBound =
+    Lifetime:{ lt:LIFETIME ":" bounds:LifetimeBound* % "+" "+"? } |
+    Type:{ binder:ForAllBinder? ty:Type ":" bounds:TypeBound* % "+" "+"? } |
+    TypeEq:{ binder:ForAllBinder? left:Type { "=" | "==" } right:Type };
+
+LifetimeBound = outlives:LIFETIME;
+TypeBound =
+    Outlives:LIFETIME |
+    Trait:TypeTraitBound |
+    TraitParen:{ "(" bound:TypeTraitBound ")" };
+TypeTraitBound = unbound:"?"? binder:ForAllBinder? path:Path;
+
+GenericArgs =
+    AngleBracket:{ "<" args_and_bindings:AngleBracketGenericArgsAndBindings? ","? ">" } |
+    Paren:{ "(" inputs:Type* % "," ","? ")" { "->" output:Type }? };
+
+// FIXME(eddyb) find a way to express this `A* B*` pattern better
+AngleBracketGenericArgsAndBindings =
+    Args:GenericArg+ % "," |
+    Bindings:TypeBinding+ % "," |
+    ArgsAndBindings:{ args:GenericArg+ % "," "," bindings:TypeBinding+ % "," };
+
+GenericArg =
+    Lifetime:LIFETIME |
+    Type:Type;
+TypeBinding = name:IDENT "=" ty:Type;

grammar/item.g

@@ -1,7 +1,92 @@
-ModuleContents = attrs:InnerAttr* items:ItemWithOuterAttr*;
+ModuleContents = attrs:InnerAttr* items:Item*;
 
-ItemWithOuterAttr = attrs:OuterAttr* item:Item;
-// TODO other items
-Item =
+Item = attrs:OuterAttr* vis:Vis? kind:ItemKind;
+ItemKind =
+    Use:{ "use" use_tree:UseTree ";" } |
     ExternCrate:{ "extern" "crate" name:IDENT { "as" rename:IDENT }? ";" } |
-    Use:{ "use" path:Path { "as" rename:IDENT }? ";" }; // TODO use trees
+    Mod:{ "mod" name:IDENT { ";" | "{" contents:ModuleContents "}" } } |
+    ForeignMod:{ "extern" abi:Abi "{" attrs:InnerAttr* items:ForeignItem* "}" } |
+    Static:{ "static" mutable:"mut"? name:IDENT ":" ty:Type "=" value:Expr ";" } |
+    Const:{ "const" name:IDENT ":" ty:Type "=" value:Expr ";" } |
+    Fn:{ header:FnHeader "fn" decl:FnDecl body:Block } |
+    TypeAlias:{ "type" name:IDENT generics:Generics? where_clause:WhereClause? "=" ty:Type ";" } |
+    ExistentialType:{ "existential" "type" name:IDENT generics:Generics? where_clause:WhereClause? ":" bounds:TypeBound* % "+" "+"? ";" } |
+    Enum:{ "enum" name:IDENT generics:Generics? where_clause:WhereClause? "{" variants:EnumVariant* % "," ","? "}" } |
+    Struct:{ "struct" name:IDENT generics:Generics? body:StructBody } |
+    Union:{ "union" name:IDENT generics:Generics? where_clause:WhereClause? "{" fields:RecordField* % "," ","? "}" } |
+    Trait:{
+        unsafety:"unsafe"? auto:"auto"? "trait" name:IDENT generics:Generics?
+        { ":" superbounds:TypeBound* % "+" "+"? }?
+        where_clause:WhereClause? "{" trait_items:TraitItem* "}"
+    } |
+    TraitAlias:{
+        "trait" name:IDENT generics:Generics?
+        { ":" superbounds:TypeBound* % "+" "+"? }?
+        where_clause:WhereClause? "=" bounds:TypeBound* % "+" "+"? ";"
+    } |
+    Impl:{
+        defaultness:"default"? unsafety:"unsafe"? "impl" generics:Generics?
+        { negate:"!"? trait_path:Path "for" }? ty:Type
+        where_clause:WhereClause? "{" attrs:InnerAttr* impl_items:ImplItem* "}"
+    } |
+    Macro:{ "macro" name:IDENT { "(" TOKEN_TREE* ")" }? "{" TOKEN_TREE* "}" } |
+    MacroCall:ItemMacroCall;
+
+UseTree =
+    Glob:{ prefix:UseTreePrefix? "*" } |
+    Nested:{ prefix:UseTreePrefix? "{" children:UseTree* % "," ","? "}" } |
+    Simple:{ path:Path { "as" rename:IDENT }? };
+UseTreePrefix =
+    Path:{ path:Path "::" } |
+    Global:"::";
+
+ForeignItem = attrs:OuterAttr* vis:Vis? kind:ForeignItemKind;
+ForeignItemKind =
+    Static:{ "static" mutable:"mut"? name:IDENT ":" ty:Type ";" } |
+    Fn:{ "fn" decl:FnDecl ";" } |
+    Type:{ "type" name:IDENT ";" } |
+    MacroCall:ItemMacroCall;
+
+TraitItem = attrs:OuterAttr* kind:TraitItemKind;
+TraitItemKind =
+    Const:{ "const" name:IDENT ":" ty:Type { "=" default:Expr }? ";" } |
+    Fn:{ header:FnHeader "fn" decl:FnDecl { default_body:Block | ";" } } |
+    Type:{ "type" name:IDENT generics:Generics? { ":" bounds:TypeBound* % "+" "+"? }? where_clause:WhereClause? { "=" default:Type }? ";" } |
+    MacroCall:ItemMacroCall;
+
+ImplItem = attrs:OuterAttr* defaultness:"default"? vis:Vis? kind:ImplItemKind;
+ImplItemKind =
+    Const:{ "const" name:IDENT ":" ty:Type "=" value:Expr ";" } |
+    Fn:{ header:FnHeader "fn" decl:FnDecl body:Block } |
+    Type:{ "type" name:IDENT generics:Generics? where_clause:WhereClause? "=" ty:Type ";" } |
+    ExistentialType:{ "existential" "type" name:IDENT generics:Generics? where_clause:WhereClause? ":" bounds:TypeBound* % "+" "+"? ";" } |
+    MacroCall:ItemMacroCall;
+
+FnHeader = constness:"const"? unsafety:"unsafe"? asyncness:"async"? { "extern" abi:Abi }?;
+FnDecl = name:IDENT generics:Generics? "(" args:FnArgs? ","? ")" { "->" ret_ty:Type }? where_clause:WhereClause?;
+
+// FIXME(eddyb) find a way to express this `A* B?` pattern better
+FnArgs =
+    Regular:FnArg+ % "," |
+    Variadic:"..." |
+    RegularAndVariadic:{ args:FnArg+ % "," "," "..." };
+FnArg =
+    SelfValue:{ mutable:"mut"? "self" } |
+    SelfRef:{ "&" lt:LIFETIME? mutable:"mut"? "self" } |
+    Regular:FnSigInput;
+
+EnumVariant = attrs:OuterAttr* name:IDENT kind:EnumVariantKind { "=" discr:Expr }?;
+EnumVariantKind =
+    Unit:{} |
+    Tuple:{ "(" fields:TupleField* % "," ","? ")" } |
+    Record:{ "{" fields:RecordField* % "," ","? "}" };
+
+// FIXME(eddyb) could maybe be shared more with `EnumVariantKind`?
+// The problem is the semicolons for `Unit` and `Tuple`, and the where clauses.
+StructBody =
+    Unit:{ where_clause:WhereClause? ";" } |
+    Tuple:{ "(" fields:TupleField* % "," ","? ")" where_clause:WhereClause? ";" } |
+    Record:{ where_clause:WhereClause? "{" fields:RecordField* % "," ","? "}" };
+
+TupleField = attrs:OuterAttr* vis:Vis? ty:Type;
+RecordField = attrs:OuterAttr* vis:Vis? name:IDENT ":" ty:Type;

grammar/macro.g

@@ -0,0 +1,13 @@
+MacroCall = path:Path "!" ident_input:IDENT? input:MacroInput;
+MacroInput =
+    "(" TOKEN_TREE* ")" |
+    "[" TOKEN_TREE* "]" |
+    "{" TOKEN_TREE* "}";
+
+// FIXME(eddyb) could maybe be shared more with `MacroInput`?
+// The problem is the semicolons for the `()` and `[]` cases.
+ItemMacroCall = path:Path "!" ident_input:IDENT? input:ItemMacroInput;
+ItemMacroInput =
+    "(" TOKEN_TREE* ")" ";" |
+    "[" TOKEN_TREE* "]" ";" |
+    "{" TOKEN_TREE* "}";

grammar/pat.g

@@ -0,0 +1,39 @@
+Pat =
+    Wild:"_" |
+    Literal:{ minus:"-"? lit:LITERAL } |
+    Range:{ start:PatRangeValue ".." end:PatRangeValue } |
+    RangeInclusive:{ start:PatRangeValue { "..." | "..=" } end:PatRangeValue } |
+    Binding:{ binding:Binding { "@" subpat:Pat }? } |
+    Paren:{ "(" pat:Pat ")" } |
+    Ref:{ "&" mutable:"mut"? pat:Pat } |
+    Box:{ "box" pat:Pat } |
+    Slice:{ "[" elems:SlicePatElem* % "," ","? "]" } |
+    Tuple:{ "(" fields:TuplePatField* % "," ","? ")" } |
+    Path:QPath |
+    TupleStruct:{ path:Path "(" fields:TuplePatField* % "," ","? ")" } |
+    Struct:{ path:Path "{" fields:StructPatFieldsAndEllipsis "}" } |
+    MacroCall:MacroCall;
+
+PatRangeValue =
+    Literal:{ minus:"-"? lit:LITERAL } |
+    Path:QPath;
+
+Binding = boxed:"box"? reference:"ref"? mutable:"mut"? name:IDENT;
+
+SlicePatElem =
+    Subslice:{ subpat:Pat? ".." } |
+    Pat:Pat;
+
+TuplePatField =
+    Ellipsis:".." |
+    Pat:Pat;
+
+// FIXME(eddyb) find a way to express this `A* B?` pattern better
+StructPatFieldsAndEllipsis =
+    Fields:{ fields:StructPatField* % "," ","? } |
+    Ellipsis:{ ".." } |
+    FieldsAndEllipsis:{ fields:StructPatField+ % "," "," ".." };
+
+StructPatField =
+    Shorthand:Binding |
+    Explicit:{ field:FieldName ":" pat:Pat };

grammar/path.g

@@ -1,2 +1,8 @@
-Path = global:"::"? segments:PathSegment* % "::";
-PathSegment = ident:IDENT; // TODO generics
+Path = global:"::"? path:RelativePath;
+RelativePath = segments:PathSegment+ % "::";
+PathSegment = ident:IDENT { "::"? args:GenericArgs }?;
+
+QSelf = "<" ty:Type { "as" trait_path:Path }? ">";
+QPath =
+    Unqualified:Path |
+    Qualified:{ qself:QSelf "::" path:RelativePath };

grammar/stmt.g

@@ -0,0 +1,7 @@
+Stmt =
+    Item:Item |
+    Local:{ attrs:OuterAttr* "let" pat:Pat { ":" ty:Type }? { "=" init:Expr }? ";" } |
+    Expr:Expr |
+    Semi:";";
+
+Block = "{" attrs:InnerAttr* Stmt* "}";