Simplify JSX and TSX lexers (#1492)

The JSX lexer handles interpolation by running an embedded version of
itself. This is not the manner in which other lexers handle
interpolation and so makes maintenance difficult. It also greatly
complicates subclassing the lexer (a practical concern because the TSX
lexer subclasses the JSX lexer).

To fix this, this commit rewrites the JSX lexer to simplify its
structure. In particular, by specifying `:interpol` and
`:interpol_inner` states, the need for an embedded version of the lexer
is obviated.

lib/rouge/lexers/jsx.rb

@@ -6,97 +6,85 @@ module Lexers
 
     class JSX < Javascript
       title 'JSX'
-      desc 'React JSX (https://facebook.github.io/react/)'
+      desc 'An XML-like syntax extension to JavaScript (facebook.github.io/jsx/)'
       tag 'jsx'
       aliases 'jsx', 'react'
       filenames '*.jsx'
 
       mimetypes 'text/x-jsx', 'application/x-jsx'
 
-      id = Javascript.id_regex
+      start { @html = HTML.new(options); push :expr_start }
 
-      def start_embed!
-        @embed ||= JSX.new(options)
-        @embed.reset!
-        @embed.push(:expr_start)
-        push :jsx_embed_root
-      end
-
-      def tag_token(name)
-        name[0] =~ /\p{Lower}/ ? Name::Tag : Name::Class
+      prepend :expr_start do
+        mixin :tag
       end
 
-      start { @html = HTML.new(options) }
-
-      state :jsx_tags do
-        rule %r/</, Punctuation, :jsx_element
+      state :tag do
+        rule %r/</ do
+          token Punctuation
+          push :tag_opening
+          push :element
+          push :element_name
+        end
       end
 
-      state :jsx_internal do
-        rule %r(</) do
+      state :tag_opening do
+        rule %r/<\// do
           token Punctuation
-          goto :jsx_end_tag
+          goto :element
+          push :element_name
         end
-
+        mixin :tag
         rule %r/{/ do
           token Str::Interpol
-          start_embed!
+          push :interpol
+          push :expr_start
         end
-
-        rule %r/[^<>{]+/ do
+        rule %r/[^<{]+/ do
           delegate @html
         end
-
-        mixin :jsx_tags
-      end
-
-      prepend :expr_start do
-        mixin :jsx_tags
       end
 
-      state :jsx_tag do
+      state :element do
         mixin :comments_and_whitespace
-        rule %r/#{id}/ do |m|
-          token tag_token(m[0])
+        rule %r/\/>/ do
+          token Punctuation
+          pop! 2
         end
-
-        rule %r/[.]/, Punctuation
-      end
-
-      state :jsx_end_tag do
-        mixin :jsx_tag
         rule %r/>/, Punctuation, :pop!
-      end
-
-      state :jsx_element do
-        rule %r/#{id}=/, Name::Attribute, :jsx_attribute
-        mixin :jsx_tag
-        rule %r/>/ do token Punctuation; goto :jsx_internal end
-        rule %r(/>), Punctuation, :pop!
-      end
-
-      state :jsx_attribute do
-        rule %r/"(\\[\\"]|[^"])*"/, Str::Double, :pop!
-        rule %r/'(\\[\\']|[^'])*'/, Str::Single, :pop!
         rule %r/{/ do
           token Str::Interpol
-          pop!
-          start_embed!
+          push :interpol
+          push :expr_start
         end
+        rule %r/\w+/, Name::Attribute
+        rule %r/=/, Punctuation
+        rule %r/(["']).*?(\1)/, Str
       end
 
-      state :jsx_embed_root do
-        rule %r/[.][.][.]/, Punctuation
+      state :element_name do
+        rule %r/[A-Z]\w*/, Name::Class
+        rule %r/\w+/, Name::Tag
+        rule %r/\./, Punctuation
+        rule(//) { pop! }
+      end
+
+      state :interpol do
         rule %r/}/, Str::Interpol, :pop!
-        mixin :jsx_embed
+        rule %r/{/ do
+          token Punctuation
+          push :interpol_inner
+          push :statement
+        end
+        mixin :root
       end
 
-      state :jsx_embed do
-        rule %r/{/ do delegate @embed; push :jsx_embed end
-        rule %r/}/ do delegate @embed; pop! end
-        rule %r/[^{}]+/ do
-          delegate @embed
+      state :interpol_inner do
+        rule %r/}/ do
+          token Punctuation
+          goto :statement
         end
+        mixin :root
       end
     end
   end

lib/rouge/lexers/tsx.rb

@@ -9,8 +9,8 @@ module Lexers
     class TSX < JSX
       extend TypescriptCommon
 
-      title 'TypeScript'
-      desc 'tsx'
+      title 'TSX'
+      desc 'TypeScript-compatible JSX (www.typescriptlang.org/docs/handbook/jsx.html)'
 
       tag 'tsx'
       filenames '*.tsx'

spec/visual/samples/jsx

@@ -1,8 +1,10 @@
+<header className="App-header">
+  Hello React!
+</header>
+
 var myDivElement = <div className="foo" />;
 ReactDOM.render(myDivElement, document.getElementById('example'));
 
-
-
 var MyComponent = React.createClass({/*...*/});
 var myElement = <MyComponent someProperty={true} />;
 ReactDOM.render(myElement, document.getElementById('example'));
@@ -50,15 +52,6 @@ var content = (
   </Nav>
 );
 
-var App = (
-  <Form>
-    <FormRow>
-      <FormLabel />
-      <FormInput />
-    </FormRow>
-  </Form>
-);
-
 var thing = <A b={function() { var c = <D e={true}>&quot;</D>; }()}/>
 
 class LikeButton extends React.Component {