A library to generate syntax diagrams for Rust macros.

Saw the call for participation on TWIR for issue #17, so decided to give it a shot. Passes tests, but seems to be slower than the current syn 0.14-based parser.

This macro should parse, but doesn't:

macro_rules! java_inner {
    (toplevel {}) => { ... };
    (toplevel { package $name:ident; $($remaining:tt)* }) => { ... };
    (toplevel { public class $name:ident {
        $($kind:ident $var:ident;)*
        ---
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (toplevel { public class $name:ident {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (class($class:ident) {}) => { ... };
    (class($class:ident) { public static void main(String[] $args:ident) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (class($class:ident) { public $constructor:ident($self:ident$(, $kind:ident $var:ident)*) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (class($class:ident) { public $ret:ident $fn:ident($self:ident$(, $kind:ident $var:ident)*) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (class($class:ident) { public static $ret:ident $fn:ident($($kind:ident $var:ident),*) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (stmt($class:ident) {}) => { ... };
    (stmt($class:ident) { System.out.println($($out:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { System.out.println_debug($($out:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { System.out.print($($out:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $kind:ident $name:ident = ($($value:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $kind:ident $name:ident = $value:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { ($name:expr) = ($($val:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { ($name:expr) = $val:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $name:ident = $val:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { ($name:expr) += $val:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { ($name:expr) -= $val:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $name:ident++; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $name:ident--; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { return ($($val:tt)*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { return $val:expr; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { break; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { continue; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { for (($($pre:tt)*); ($($cond:tt)*); ($($post:tt)*)) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { if ($($cond:tt)*) {
        $($success:tt)*
    } $(else if ($($elseif_cond:tt)*) {
        $($elseif_success:tt)*
    // Else is not optional but we can use * as a hack
    })* $(else {
        $($otherwise:tt)*
    })*; $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { switch($($search:tt)*) {
        $(case ($match:expr) {
            $($success:tt)*
        })*
        // Should only be one default but rust doesn't have optional macro args yet AFAIK
        $(default {
            $($default:tt)*
        })*
    } $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { while ($($cond:tt)*) {
        $($inner:tt)*
    } $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $val:ident.$fn:ident($(($($var:tt)*)),*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $val:ident.$fn:ident($($var:expr),*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $fn:ident($(($($var:tt)*)),*); $($remaining:tt)* }) => { ... };
    (stmt($class:ident) { $fn:ident($($var:expr),*); $($remaining:tt)* }) => { ... };
    (expr($class:ident) { $array:ident[$index:expr] }) => { ... };
    (expr($class:ident) { $array:ident.length }) => { ... };
    (expr($class:ident) { ($($var1:tt)*) $(+ ($($var2:tt)*))+ }) => { ... };
    (expr($class:ident) { $var:ident++ }) => { ... };
    (expr($class:ident) { $var1:ident $op:tt $var2:ident }) => { ... };
    (expr($class:ident) { ($($var1:tt)*) $op:tt ($($var2:tt)*) }) => { ... };
    (expr($_class:ident) { new $class:ident($(($($var:tt)*)),*) }) => { ... };
    (expr($_class:ident) { new $class:ident($($var:expr),*) }) => { ... };
    (expr($class:ident) { $fn:ident($(($($var:tt)*)),*) }) => { ... };
    (expr($class:ident) { $fn:ident($($var:expr),*) }) => { ... };
    (expr($class:ident) { $expr:expr }) => { ... };
    (kind byte) => { ... };
    (kind short) => { ... };
    (kind int) => { ... };
    (kind long) => { ... };
    (kind void) => { ... };
    (kind $name:ident) => { ... };
}

I've tried to paste the (internal) diesel table marco into the web demo. This fails with the following error:

Failed to parse, and I didn't even write an error-handler. Anyway:
ParseError(None)

The issue seems to be with the following rule:

macro_rules! __diesel_parse_table {
    // Found an import
    (
        tokens = [use $($import:tt)::+; $($rest:tt)*],
        imports = [$($imports:tt)*],
        $($args:tt)*
    ) => {
        __diesel_parse_table! {
            tokens = [$($rest)*],
            imports = [$($imports)* use $($import)::+;],
            $($args)*
        }
    };
}

If you generate a syntax diagram for this macro:

macro_rules! isolate_params {
    ($self:ident.$fn:ident(&self $(, $ident:ident: $ty:ty)*)) => ($self.$fn($($ident),*));
    ($self:ident.$fn:ident(&$($lt:tt)? self $(, $ident:ident: $ty:ty)*)) => ($self.$fn($($ident),*));
}

The right-hand part of the diagram doesn't get properly collapsed down, resulting in a diagram that looks like this:

If you simplify the macro a bit, the diagram generator behaves correctly. I've removed the $self:ident.$fn:ident prefix here, and the generator behaves as follows:

macro_rules! isolate_params {
    (&self $(, $ident:ident: $ty:ty)*) => ($self.$fn($($ident),*));
    (&$($lt:tt)? self $(, $ident:ident: $ty:ty)*) => ($self.$fn($($ident),*));
}

Which is more in line with what I'd expect to see in the first case.

As an aside to this issue: thank you for making this! Having this around has been super helpful for debugging my own macros, and produces output that's much easier to reason about than the trace_macros stuff built into the compiler.

// We actually keep Groups as the parser sees them, representing them
// visually as boxed (literally "boxed", like a rectangle of pixels) sequence.
// This has two effects, both of which probably desireable:
//  * Groups are represented as a visually bound sequence of (nested) elements.
//  * The optimizer does not peek into a group but sees it as a whole.
//    This means we never tear apart a group while merging common tails.
//    While one might consider this correct-ish, it may prevent a *lot*
//    of folding if the macro is very large.
//  Maybe add a knob? Maybe this is superficial?

When pasting the code for the map_for macro into the wasm demo, I get:

Failed to parse, and I didn't even write an error-handler. Anyway:

ParseError(None)

However, the same version of map_for works with the latest Rust compiler (v1.28.0).

The following macro causes the folding-pass to trigger an assertion error.

macro_rules! Token {
    (abstract) => { ... };
    (as) => { ... };
    (async) => { ... };
    (auto) => { ... };
    (become) => { ... };
    (box) => { ... };
    (break) => { ... };
    (const) => { ... };
    (continue) => { ... };
    (crate) => { ... };
    (default) => { ... };
    (do) => { ... };
    (dyn) => { ... };
    (else) => { ... };
    (enum) => { ... };
    (existential) => { ... };
    (extern) => { ... };
    (final) => { ... };
    (fn) => { ... };
    (for) => { ... };
    (if) => { ... };
    (impl) => { ... };
    (in) => { ... };
    (let) => { ... };
    (loop) => { ... };
    (macro) => { ... };
    (match) => { ... };
    (mod) => { ... };
    (move) => { ... };
    (mut) => { ... };
    (override) => { ... };
    (priv) => { ... };
    (pub) => { ... };
    (ref) => { ... };
    (return) => { ... };
    (Self) => { ... };
    (self) => { ... };
    (static) => { ... };
    (struct) => { ... };
    (super) => { ... };
    (trait) => { ... };
    (try) => { ... };
    (type) => { ... };
    (typeof) => { ... };
    (union) => { ... };
    (unsafe) => { ... };
    (unsized) => { ... };
    (use) => { ... };
    (virtual) => { ... };
    (where) => { ... };
    (while) => { ... };
    (yield) => { ... };
    (+) => { ... };
    (+=) => { ... };
    (&) => { ... };
    (&&) => { ... };
    (&=) => { ... };
    (@) => { ... };
    (!) => { ... };
    (^) => { ... };
    (^=) => { ... };
    (:) => { ... };
    (::) => { ... };
    (,) => { ... };
    (/) => { ... };
    (/=) => { ... };
    (.) => { ... };
    (..) => { ... };
    (...) => { ... };
    (..=) => { ... };
    (=) => { ... };
    (==) => { ... };
    (>=) => { ... };
    (>) => { ... };
    (<=) => { ... };
    (<) => { ... };
    (*=) => { ... };
    (!=) => { ... };
    (|) => { ... };
    (|=) => { ... };
    (||) => { ... };
    (#) => { ... };
    (?) => { ... };
    (->) => { ... };
    (<-) => { ... };
    (%) => { ... };
    (%=) => { ... };
    (=>) => { ... };
    (;) => { ... };
    (<<) => { ... };
    (<<=) => { ... };
    (>>) => { ... };
    (>>=) => { ... };
    (*) => { ... };
    (-) => { ... };
    (-=) => { ... };
    (~) => { ... };
    (_) => { ... };
}```

This is not accepted by rustc.

macro_rules! m { ($($tt:tt)-?) => {} }

error: the `?` macro repetition operator does not take a separator
 --> src/lib.rs
  |
  | macro_rules! m { ($($tt:tt)-?) => {} }
  |                            ^

Our parser for macro_rules!()-syntax uses syn, which has recently bumped to 0.15, bringing a lot of changes. Rewrite the parser for syn-0.15.

The current parser for macro_rules!()-bodies is using syn-0.14. It does have some unresolved issues (1,2,3) which need fixing anyway.

One can use the existing tests (cargo test) as the last line of defense - those should parse in any case. A more comprehensive test is accessed via cargo test -- --ignored, which parses, builds and verifies a large number of macros. One can also use cargo run --example various to quickly generate a large number of examples.

In many macros, multiple branches share a lot of similar sub-patterns. It would be easier to visually read the diagrams if the similar sub-patterns were aligned. E.g. aligning the 4 occurrences of macro_name ! into one column in this example, and also aligning the 2 occurrences of values = vf ,.

And for macros where a sub-patterns appears verbatim in multiple branches of the macro (or multiple times in one branch), it would make sense to highlight that they are not just similar but the same, by factoring out that sub-pattern into its own sub-rule diagram (and then inserting the name of the sub-rule (or an arrow to the sub-rule diagram) in place of the occurrences of the sub-pattern).

The foldcommontails-walker currently misses some opportunities as demonstrated here (the optional mut should be folded) and here (the pub should be folded).

The problem most likely is that the optimizer might produce a Choice of just one element, when it effectivly becomes a Sequence. Since it can not "look into" a Choice when evaluating common tails, it misses out.

The fix is probably to implement a simplifying pass that

• Turns Choice of exactly one element into Sequence
• Turns Sequence of exactly one element into that element
• Removes the needless Sequence in a Repeat
• and such.

There are situations where we do this manually now; get rid of them.

The result of the folding pass should be simplified before re-evaluating the current Choice for more opportunities. The overall result should also be simplified.

Currently the instructions/docs/reference screen on the right takes half of horizontal dimension, and macro input has only 50% left. Please consider a toggle/collapse/tab for the instructions area.

A workaround in Firefox:

• install addons.mozilla.org/en-US/firefox/addon/styl-us
• in Stylus, enter a style:

/* ==UserStyle==
@name           lukaslueg.github.io - 2022-05-20, 10:46:44 a.m.
@namespace      github.com/openstyles/stylus
@version        1.0.0
@description    A new userstyle
@author         Me
==/UserStyle== */

@-moz-document url-prefix("https://lukaslueg.github.io/macro_railroad_wasm_demo/") {
    div#copytext {
        display: none !important;
    }
    div#content {
        grid-template-columns: 1fr 0fr !important;
    }
}

See also lukaslueg/railroad#3.

The SVGs generated by macro_railroad should display correctly on most modern browsers. Although we do not use too much CSS-trickery, we want to ensure that the most used browsers always render the same output. This is especially important if macro_railroad should ever get on a path to be included in some form into rustdoc.

Come up with a scheme to automatically test that SVGs generated for specified test-cases result in the same rendering across browsers.

A rough sketch could be a test-suite that takes certain macros as input, produces variants of the diagrams, sends them to selenium to produce a screenshot and ensure that all screenshots look the same (for some definition).

Rust's RFC1985 has a list of browsers that rustdoc should always support. As of now this is

last 2 Chrome versions last 1 Firefox version Firefox ESR last 1 Safari version last 1 iOS version last 1 Edge version last 1 UCAndroid version

There is currently no way in Rust itself to have per-nonterminal documentation. For example, in nom4::method, there is just no way to express what a, _self and args actually do.

One could come up with a convention to document this in a standardized way that does not hurt rustdoc, rustc or readability but can be parsed and use as tooltips, legend or something else in these diagrams.

Cooked up example:

/// This is the panic-macro. It causes panic.
///
/// $msg:expr Something that evaluates to a string, may be printed to stderr if possible
/// $fmt::expr A format string, as used `::std::fmt`
/// $arg::tt Arguments to the format string
macro_rules! panic {
    () => { ... };
    ($msg:expr) => { ... };
    ($msg:expr,) => { ... };
    ($fmt:expr, $($arg:tt)+) => { ... };
}

We could associate that comment, parse the mini-language and use the hints as further explanation. Note that each name:fragment can only appear once, as macro_railroads already implies that a name:fragment-tuple carries the same meaning throughout the macro definition.

A diagram's geometry is separate from it's fine-tuned design via CSS. The current default CSS is probably not the best.

Come up with a better CSS-theme for macro_railroad.

A dark theme would probably be useful.

It would be extremely useful to bidirectionally map span information to diagram elements so that

• we can highlight sections in the source-text which correspond to a diagram-element. The user might pick (mouseover) an element; the corresponding sections in the source should be highlighted; there may be multiple such sections in different arms (think of the "vertex" terminal in gfx_defines, which in the optimized form appears once but has two sources).
• we can highlight sections of the diagram which correspond to a syntax-element. The user might select parts of the text. The diagram-elements which are sourced by that part of the text should be highlighted.

This requires us to acquire, collect and map CodeSpan-information up to the point where we generate a diagram. The code to add custom data-attributes to an SVG-element is already in place. In addition to a diagram, we also give back some form of code-map. Some JS-code can then control the blinkenlights via CSS-rules regarding the affected SVG-nodes.