Clean up readNumber, fix parsing of '2.+2'

Issue #9
2013-01-30 16:38:31 +01:00
parent 792fa96e93
commit 87a6a5a8cd
3 changed files with 61 additions and 30 deletions
--- a/acorn.js
+++ b/acorn.js
@@ -536,9 +536,9 @@
  // The `forceRegexp` parameter is used in the one case where the
  // `tokRegexpAllowed` trick does not work. See `parseStatement`.

-  function readToken_dot(code) {
+  function readToken_dot() {
    var next = input.charCodeAt(tokPos+1);
-    if (next >= 48 && next <= 57) return readNumber(String.fromCharCode(code));
+    if (next >= 48 && next <= 57) return readNumber(true);
    ++tokPos;
    return finishToken(_dot);
  }
@@ -600,7 +600,7 @@
      // The interpretation of a dot depends on whether it is followed
      // by a digit.
    case 46: // '.'
-      return readToken_dot(code);
+      return readToken_dot();

      // Punctuation tokens.
    case 40: ++tokPos; return finishToken(_parenL);
@@ -621,7 +621,7 @@
      // Anything else beginning with a digit is an integer, octal
      // number, or float.
    case 49: case 50: case 51: case 52: case 53: case 54: case 55: case 56: case 57: // 1-9
-      return readNumber(String.fromCharCode(code));
+      return readNumber(false);

      // Quotes produce strings.
    case 34: case 39: // '"', "'"
@@ -746,18 +746,18 @@

  // Read an integer, octal integer, or floating-point number.
  
-  function readNumber(ch) {
-    var start = tokPos, isFloat = ch === ".";
-    if (!isFloat && readInt(10) == null) raise(start, "Invalid number");
-    if (isFloat || input.charAt(tokPos) === ".") {
-      var next = input.charAt(++tokPos);
-      if (next === "-" || next === "+") ++tokPos;
-      if (readInt(10) === null && ch === ".") raise(start, "Invalid number");
+  function readNumber(startsWithDot) {
+    var start = tokPos, isFloat = false, octal = input.charCodeAt(tokPos) === 48;
+    if (!startsWithDot && readInt(10) === null) raise(start, "Invalid number");
+    if (input.charCodeAt(tokPos) === 46) {
+      ++tokPos;
+      readInt(10);
      isFloat = true;
    }
-    if (/e/i.test(input.charAt(tokPos))) {
-      var next = input.charAt(++tokPos);
-      if (next === "-" || next === "+") ++tokPos;
+    var next = input.charCodeAt(tokPos);
+    if (next === 69 || next === 101) { // 'eE'
+      next = input.charCodeAt(++tokPos);
+      if (next === 43 || next === 45) ++tokPos; // '+-'
      if (readInt(10) === null) raise(start, "Invalid number")
      isFloat = true;
    }
@@ -765,7 +765,7 @@

    var str = input.slice(start, tokPos), val;
    if (isFloat) val = parseFloat(str);
-    else if (ch !== "0" || str.length === 1) val = parseInt(str, 10);
+    else if (!octal || str.length === 1) val = parseInt(str, 10);
    else if (/[89]/.test(str) || strict) raise(start, "Invalid number");
    else val = parseInt(str, 8);
    return finishToken(_num, val);
--- a/index.html
+++ b/index.html
@@ -341,9 +341,9 @@ into it.</p>
 <p>All in the name of speed.</p>

 <p>The <code>forceRegexp</code> parameter is used in the one case where the
-<code>tokRegexpAllowed</code> trick does not work. See <code>parseStatement</code>.</p>             </td>             <td class="code">               <div class="highlight"><pre>  <span class="kd">function</span> <span class="nx">readToken_dot</span><span class="p">(</span><span class="nx">code</span><span class="p">)</span> <span class="p">{</span>
+<code>tokRegexpAllowed</code> trick does not work. See <code>parseStatement</code>.</p>             </td>             <td class="code">               <div class="highlight"><pre>  <span class="kd">function</span> <span class="nx">readToken_dot</span><span class="p">()</span> <span class="p">{</span>
    <span class="kd">var</span> <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="o">+</span><span class="mi">1</span><span class="p">);</span>
-    <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">&gt;=</span> <span class="mi">48</span> <span class="o">&amp;&amp;</span> <span class="nx">next</span> <span class="o">&lt;=</span> <span class="mi">57</span><span class="p">)</span> <span class="k">return</span> <span class="nx">readNumber</span><span class="p">(</span><span class="nb">String</span><span class="p">.</span><span class="nx">fromCharCode</span><span class="p">(</span><span class="nx">code</span><span class="p">));</span>
+    <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">&gt;=</span> <span class="mi">48</span> <span class="o">&amp;&amp;</span> <span class="nx">next</span> <span class="o">&lt;=</span> <span class="mi">57</span><span class="p">)</span> <span class="k">return</span> <span class="nx">readNumber</span><span class="p">(</span><span class="kc">true</span><span class="p">);</span>
    <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span>
    <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_dot</span><span class="p">);</span>
  <span class="p">}</span>
@@ -403,7 +403,7 @@ into it.</p>
  <span class="kd">function</span> <span class="nx">getTokenFromCode</span><span class="p">(</span><span class="nx">code</span><span class="p">)</span> <span class="p">{</span>
    <span class="k">switch</span><span class="p">(</span><span class="nx">code</span><span class="p">)</span> <span class="p">{</span></pre></div>             </td>           </tr>                               <tr id="section-57">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-57">&#182;</a>               </div>               <p>The interpretation of a dot depends on whether it is followed
 by a digit.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">46</span><span class="o">:</span> <span class="c1">// &#39;.&#39;</span>
-      <span class="k">return</span> <span class="nx">readToken_dot</span><span class="p">(</span><span class="nx">code</span><span class="p">);</span></pre></div>             </td>           </tr>                               <tr id="section-58">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-58">&#182;</a>               </div>               <p>Punctuation tokens.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">40</span><span class="o">:</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_parenL</span><span class="p">);</span>
+      <span class="k">return</span> <span class="nx">readToken_dot</span><span class="p">();</span></pre></div>             </td>           </tr>                               <tr id="section-58">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-58">&#182;</a>               </div>               <p>Punctuation tokens.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">40</span><span class="o">:</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_parenL</span><span class="p">);</span>
    <span class="k">case</span> <span class="mi">41</span><span class="o">:</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_parenR</span><span class="p">);</span>
    <span class="k">case</span> <span class="mi">59</span><span class="o">:</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_semi</span><span class="p">);</span>
    <span class="k">case</span> <span class="mi">44</span><span class="o">:</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_comma</span><span class="p">);</span>
@@ -416,7 +416,7 @@ by a digit.</p>             </td>             <td class="code">               <d
      <span class="kd">var</span> <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="o">+</span><span class="mi">1</span><span class="p">);</span>
      <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">===</span> <span class="mi">120</span> <span class="o">||</span> <span class="nx">next</span> <span class="o">===</span> <span class="mi">88</span><span class="p">)</span> <span class="k">return</span> <span class="nx">readHexNumber</span><span class="p">();</span></pre></div>             </td>           </tr>                               <tr id="section-60">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-60">&#182;</a>               </div>               <p>Anything else beginning with a digit is an integer, octal
 number, or float.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">49</span><span class="o">:</span> <span class="k">case</span> <span class="mi">50</span><span class="o">:</span> <span class="k">case</span> <span class="mi">51</span><span class="o">:</span> <span class="k">case</span> <span class="mi">52</span><span class="o">:</span> <span class="k">case</span> <span class="mi">53</span><span class="o">:</span> <span class="k">case</span> <span class="mi">54</span><span class="o">:</span> <span class="k">case</span> <span class="mi">55</span><span class="o">:</span> <span class="k">case</span> <span class="mi">56</span><span class="o">:</span> <span class="k">case</span> <span class="mi">57</span><span class="o">:</span> <span class="c1">// 1-9</span>
-      <span class="k">return</span> <span class="nx">readNumber</span><span class="p">(</span><span class="nb">String</span><span class="p">.</span><span class="nx">fromCharCode</span><span class="p">(</span><span class="nx">code</span><span class="p">));</span></pre></div>             </td>           </tr>                               <tr id="section-61">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-61">&#182;</a>               </div>               <p>Quotes produce strings.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">34</span><span class="o">:</span> <span class="k">case</span> <span class="mi">39</span><span class="o">:</span> <span class="c1">// &#39;&quot;&#39;, &quot;&#39;&quot;</span>
+      <span class="k">return</span> <span class="nx">readNumber</span><span class="p">(</span><span class="kc">false</span><span class="p">);</span></pre></div>             </td>           </tr>                               <tr id="section-61">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-61">&#182;</a>               </div>               <p>Quotes produce strings.</p>             </td>             <td class="code">               <div class="highlight"><pre>    <span class="k">case</span> <span class="mi">34</span><span class="o">:</span> <span class="k">case</span> <span class="mi">39</span><span class="o">:</span> <span class="c1">// &#39;&quot;&#39;, &quot;&#39;&quot;</span>
      <span class="k">return</span> <span class="nx">readString</span><span class="p">(</span><span class="nx">code</span><span class="p">);</span></pre></div>             </td>           </tr>                               <tr id="section-62">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-62">&#182;</a>               </div>               <p>Operators are parsed inline in tiny state machines. '=' (61) is
 often referred to. <code>finishOp</code> simply skips the amount of
 characters it is given as second argument, and returns a token
@@ -517,18 +517,18 @@ will return <code>null</code> unless the integer has exactly <code>len</code> di
    <span class="k">if</span> <span class="p">(</span><span class="nx">isIdentifierStart</span><span class="p">(</span><span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">)))</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">,</span> <span class="s2">&quot;Identifier directly after number&quot;</span><span class="p">);</span>
    <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_num</span><span class="p">,</span> <span class="nx">val</span><span class="p">);</span>
  <span class="p">}</span></pre></div>             </td>           </tr>                               <tr id="section-68">             <td class="docs">               <div class="pilwrap">                 <a class="pilcrow" href="#section-68">&#182;</a>               </div>               <p>Read an integer, octal integer, or floating-point number.</p>             </td>             <td class="code">               <div class="highlight"><pre>  
-  <span class="kd">function</span> <span class="nx">readNumber</span><span class="p">(</span><span class="nx">ch</span><span class="p">)</span> <span class="p">{</span>
-    <span class="kd">var</span> <span class="nx">start</span> <span class="o">=</span> <span class="nx">tokPos</span><span class="p">,</span> <span class="nx">isFloat</span> <span class="o">=</span> <span class="nx">ch</span> <span class="o">===</span> <span class="s2">&quot;.&quot;</span><span class="p">;</span>
-    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">isFloat</span> <span class="o">&amp;&amp;</span> <span class="nx">readInt</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> <span class="o">==</span> <span class="kc">null</span><span class="p">)</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="s2">&quot;Invalid number&quot;</span><span class="p">);</span>
-    <span class="k">if</span> <span class="p">(</span><span class="nx">isFloat</span> <span class="o">||</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">)</span> <span class="o">===</span> <span class="s2">&quot;.&quot;</span><span class="p">)</span> <span class="p">{</span>
-      <span class="kd">var</span> <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="o">++</span><span class="nx">tokPos</span><span class="p">);</span>
-      <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">===</span> <span class="s2">&quot;-&quot;</span> <span class="o">||</span> <span class="nx">next</span> <span class="o">===</span> <span class="s2">&quot;+&quot;</span><span class="p">)</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span>
-      <span class="k">if</span> <span class="p">(</span><span class="nx">readInt</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> <span class="o">===</span> <span class="kc">null</span> <span class="o">&amp;&amp;</span> <span class="nx">ch</span> <span class="o">===</span> <span class="s2">&quot;.&quot;</span><span class="p">)</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="s2">&quot;Invalid number&quot;</span><span class="p">);</span>
+  <span class="kd">function</span> <span class="nx">readNumber</span><span class="p">(</span><span class="nx">startsWithDot</span><span class="p">)</span> <span class="p">{</span>
+    <span class="kd">var</span> <span class="nx">start</span> <span class="o">=</span> <span class="nx">tokPos</span><span class="p">,</span> <span class="nx">isFloat</span> <span class="o">=</span> <span class="kc">false</span><span class="p">,</span> <span class="nx">octal</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">)</span> <span class="o">===</span> <span class="mi">48</span><span class="p">;</span>
+    <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">startsWithDot</span> <span class="o">&amp;&amp;</span> <span class="nx">readInt</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> <span class="o">===</span> <span class="kc">null</span><span class="p">)</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="s2">&quot;Invalid number&quot;</span><span class="p">);</span>
+    <span class="k">if</span> <span class="p">(</span><span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">)</span> <span class="o">===</span> <span class="mi">46</span><span class="p">)</span> <span class="p">{</span>
+      <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span>
+      <span class="nx">readInt</span><span class="p">(</span><span class="mi">10</span><span class="p">);</span>
      <span class="nx">isFloat</span> <span class="o">=</span> <span class="kc">true</span><span class="p">;</span>
    <span class="p">}</span>
-    <span class="k">if</span> <span class="p">(</span><span class="sr">/e/i</span><span class="p">.</span><span class="nx">test</span><span class="p">(</span><span class="nx">input</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">)))</span> <span class="p">{</span>
-      <span class="kd">var</span> <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charAt</span><span class="p">(</span><span class="o">++</span><span class="nx">tokPos</span><span class="p">);</span>
-      <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">===</span> <span class="s2">&quot;-&quot;</span> <span class="o">||</span> <span class="nx">next</span> <span class="o">===</span> <span class="s2">&quot;+&quot;</span><span class="p">)</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span>
+    <span class="kd">var</span> <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="nx">tokPos</span><span class="p">);</span>
+    <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">===</span> <span class="mi">69</span> <span class="o">||</span> <span class="nx">next</span> <span class="o">===</span> <span class="mi">101</span><span class="p">)</span> <span class="p">{</span> <span class="c1">// &#39;eE&#39;</span>
+      <span class="nx">next</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">charCodeAt</span><span class="p">(</span><span class="o">++</span><span class="nx">tokPos</span><span class="p">);</span>
+      <span class="k">if</span> <span class="p">(</span><span class="nx">next</span> <span class="o">===</span> <span class="mi">43</span> <span class="o">||</span> <span class="nx">next</span> <span class="o">===</span> <span class="mi">45</span><span class="p">)</span> <span class="o">++</span><span class="nx">tokPos</span><span class="p">;</span> <span class="c1">// &#39;+-&#39;</span>
      <span class="k">if</span> <span class="p">(</span><span class="nx">readInt</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> <span class="o">===</span> <span class="kc">null</span><span class="p">)</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="s2">&quot;Invalid number&quot;</span><span class="p">)</span>
      <span class="nx">isFloat</span> <span class="o">=</span> <span class="kc">true</span><span class="p">;</span>
    <span class="p">}</span>
@@ -536,7 +536,7 @@ will return <code>null</code> unless the integer has exactly <code>len</code> di

    <span class="kd">var</span> <span class="nx">str</span> <span class="o">=</span> <span class="nx">input</span><span class="p">.</span><span class="nx">slice</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="nx">tokPos</span><span class="p">),</span> <span class="nx">val</span><span class="p">;</span>
    <span class="k">if</span> <span class="p">(</span><span class="nx">isFloat</span><span class="p">)</span> <span class="nx">val</span> <span class="o">=</span> <span class="nb">parseFloat</span><span class="p">(</span><span class="nx">str</span><span class="p">);</span>
-    <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="nx">ch</span> <span class="o">!==</span> <span class="s2">&quot;0&quot;</span> <span class="o">||</span> <span class="nx">str</span><span class="p">.</span><span class="nx">length</span> <span class="o">===</span> <span class="mi">1</span><span class="p">)</span> <span class="nx">val</span> <span class="o">=</span> <span class="nb">parseInt</span><span class="p">(</span><span class="nx">str</span><span class="p">,</span> <span class="mi">10</span><span class="p">);</span>
+    <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="nx">octal</span> <span class="o">||</span> <span class="nx">str</span><span class="p">.</span><span class="nx">length</span> <span class="o">===</span> <span class="mi">1</span><span class="p">)</span> <span class="nx">val</span> <span class="o">=</span> <span class="nb">parseInt</span><span class="p">(</span><span class="nx">str</span><span class="p">,</span> <span class="mi">10</span><span class="p">);</span>
    <span class="k">else</span> <span class="k">if</span> <span class="p">(</span><span class="sr">/[89]/</span><span class="p">.</span><span class="nx">test</span><span class="p">(</span><span class="nx">str</span><span class="p">)</span> <span class="o">||</span> <span class="nx">strict</span><span class="p">)</span> <span class="nx">raise</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="s2">&quot;Invalid number&quot;</span><span class="p">);</span>
    <span class="k">else</span> <span class="nx">val</span> <span class="o">=</span> <span class="nb">parseInt</span><span class="p">(</span><span class="nx">str</span><span class="p">,</span> <span class="mi">8</span><span class="p">);</span>
    <span class="k">return</span> <span class="nx">finishToken</span><span class="p">(</span><span class="nx">_num</span><span class="p">,</span> <span class="nx">val</span><span class="p">);</span>
--- a/test/tests.js
+++ b/test/tests.js
@@ -25987,6 +25987,37 @@ test("123..toString(10)", {
  ]
 });

+test("123.+2", {
+  type: "Program",
+  start: 0,
+  end: 6,
+  body: [
+    {
+      type: "ExpressionStatement",
+      start: 0,
+      end: 6,
+      expression: {
+        type: "BinaryExpression",
+        start: 0,
+        left: {
+          type: "Literal",
+          start: 0,
+          end: 4,
+          value: 123
+        },
+        operator: "+",
+        right: {
+          type: "Literal",
+          start: 5,
+          end: 6,
+          value: 2
+        },
+        end: 6
+      }
+    }
+  ]
+});
+
 test("a\u2028b", {
  type: "Program",
  start: 0,