1 /*
2 * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.nashorn.internal.parser;
27
28 import static jdk.nashorn.internal.parser.TokenType.ADD;
29 import static jdk.nashorn.internal.parser.TokenType.DECIMAL;
30 import static jdk.nashorn.internal.parser.TokenType.EOF;
31 import static jdk.nashorn.internal.parser.TokenType.EOL;
32 import static jdk.nashorn.internal.parser.TokenType.ERROR;
33 import static jdk.nashorn.internal.parser.TokenType.ESCSTRING;
34 import static jdk.nashorn.internal.parser.TokenType.EXECSTRING;
35 import static jdk.nashorn.internal.parser.TokenType.FLOATING;
36 import static jdk.nashorn.internal.parser.TokenType.HEXADECIMAL;
37 import static jdk.nashorn.internal.parser.TokenType.LBRACE;
38 import static jdk.nashorn.internal.parser.TokenType.LPAREN;
39 import static jdk.nashorn.internal.parser.TokenType.OCTAL;
40 import static jdk.nashorn.internal.parser.TokenType.RBRACE;
41 import static jdk.nashorn.internal.parser.TokenType.REGEX;
42 import static jdk.nashorn.internal.parser.TokenType.RPAREN;
43 import static jdk.nashorn.internal.parser.TokenType.STRING;
44 import static jdk.nashorn.internal.parser.TokenType.XML;
45
46 import jdk.nashorn.internal.runtime.ECMAErrors;
47 import jdk.nashorn.internal.runtime.ErrorManager;
48 import jdk.nashorn.internal.runtime.JSErrorType;
49 import jdk.nashorn.internal.runtime.JSType;
50 import jdk.nashorn.internal.runtime.ParserException;
51 import jdk.nashorn.internal.runtime.Source;
52 import jdk.nashorn.internal.runtime.options.Options;
53
54 /**
55 * Responsible for converting source content into a stream of tokens.
56 *
57 */
58 @SuppressWarnings("fallthrough")
59 public class Lexer extends Scanner {
60 private static final long MIN_INT_L = Integer.MIN_VALUE;
61 private static final long MAX_INT_L = Integer.MAX_VALUE;
62
63 private static final boolean XML_LITERALS = Options.getBooleanProperty("nashorn.lexer.xmlliterals");
64
65 /** Content source. */
66 private final Source source;
67
68 /** Buffered stream for tokens. */
69 private final TokenStream stream;
70
71 /** True if here and edit strings are supported. */
72 private final boolean scripting;
73
74 /** True if a nested scan. (scan to completion, no EOF.) */
75 private final boolean nested;
76
77 /** Pending new line number and position. */
78 private int pendingLine;
79
80 /** Position of last EOL + 1. */
81 private int linePosition;
82
83 /** Type of last token added. */
84 private TokenType last;
85
86 private static final String JAVASCRIPT_WHITESPACE;
87 private static final String JAVASCRIPT_WHITESPACE_EOL;
88 private static final String JAVASCRIPT_WHITESPACE_IN_REGEXP;
89
90 private static final String JSON_WHITESPACE;
91 private static final String JSON_WHITESPACE_EOL;
92
93 static String unicodeEscape(final char ch) {
94 final StringBuilder sb = new StringBuilder();
95
96 sb.append("\\u");
97
98 final String hex = Integer.toHexString(ch);
99 for (int i = hex.length(); i < 4; i++) {
100 sb.append('0');
101 }
102 sb.append(hex);
103
104 return sb.toString();
105 }
106
107 static {
108 final StringBuilder ws = new StringBuilder();
109 final StringBuilder wsEOL = new StringBuilder();
110 final StringBuilder wsRegExp = new StringBuilder();
111 final StringBuilder jsonWs = new StringBuilder();
112
113 jsonWs.append((char)0x000a);
114 jsonWs.append((char)0x000d);
115 JSON_WHITESPACE_EOL = jsonWs.toString();
116
117 jsonWs.append((char)0x0009);
118 jsonWs.append((char)0x0020);
119 JSON_WHITESPACE = jsonWs.toString();
120
121 for (int i = 0; i <= 0xffff; i++) {
122 switch (i) {
123 case 0x000a: // line feed
124 case 0x000d: // carriage return (ctrl-m)
125 case 0x2028: // line separator
126 case 0x2029: // paragraph separator
127 wsEOL.append((char)i);
128 case 0x0009: // tab
129 case 0x0020: // ASCII space
130 case 0x000b: // tabulation line
131 case 0x000c: // ff (ctrl-l)
132 case 0x00a0: // Latin-1 space
133 case 0x1680: // Ogham space mark
134 case 0x180e: // separator, Mongolian vowel
135 case 0x2000: // en quad
136 case 0x2001: // em quad
137 case 0x2002: // en space
138 case 0x2003: // em space
139 case 0x2004: // three-per-em space
140 case 0x2005: // four-per-em space
141 case 0x2006: // six-per-em space
142 case 0x2007: // figure space
143 case 0x2008: // punctuation space
144 case 0x2009: // thin space
145 case 0x200a: // hair space
146 case 0x202f: // narrow no-break space
147 case 0x205f: // medium mathematical space
148 case 0x3000: // ideographic space
149 case 0xfeff: // byte order mark
150 ws.append((char)i);
151
152 wsRegExp.append(Lexer.unicodeEscape((char)i));
153 break;
154
155 default:
156 break;
157 }
158 }
159
160 JAVASCRIPT_WHITESPACE = ws.toString();
161 JAVASCRIPT_WHITESPACE_EOL = wsEOL.toString();
162 JAVASCRIPT_WHITESPACE_IN_REGEXP = wsRegExp.toString();
163
164 }
165
166 /**
167 * Constructor
168 *
169 * @param source the source
170 * @param stream the token stream to lex
171 */
172 public Lexer(final Source source, final TokenStream stream) {
173 this(source, stream, false);
174 }
175
176 /**
177 * Constructor
178 *
179 * @param source the source
180 * @param stream the token stream to lex
181 * @param scripting are we in scripting mode
182 */
183 public Lexer(final Source source, final TokenStream stream, final boolean scripting) {
184 super(source.getContent(), 1, 0, source.getLength());
185
186 this.source = source;
187 this.stream = stream;
188 this.scripting = scripting;
189 this.nested = false;
190 this.pendingLine = 1;
191 this.last = EOL;
192 }
193
194 private Lexer(final Lexer lexer, final State state) {
195 super(lexer, state);
196
197 source = lexer.source;
198 stream = lexer.stream;
199 scripting = lexer.scripting;
200 nested = true;
201
202 pendingLine = state.pendingLine;
203 linePosition = state.linePosition;
204 last = EOL;
205 }
206
207 static class State extends Scanner.State {
208 /** Pending new line number and position. */
209 public final int pendingLine;
210
211 /** Position of last EOL + 1. */
212 public final int linePosition;
213
214 /** Type of last token added. */
215 public final TokenType last;
216
217 /*
218 * Constructor.
219 */
220
221 State(final int position, final int limit, final int line, final int pendingLine, final int linePosition, final TokenType last) {
222 super(position, limit, line);
223
224 this.pendingLine = pendingLine;
225 this.linePosition = linePosition;
226 this.last = last;
227 }
228 }
229
230 /**
231 * Save the state of the scan.
232 *
233 * @return Captured state.
234 */
235 @Override
236 State saveState() {
237 return new State(position, limit, line, pendingLine, linePosition, last);
238 }
239
240 /**
241 * Restore the state of the scan.
242 *
243 * @param state
244 * Captured state.
245 */
246 void restoreState(final State state) {
247 super.restoreState(state);
248
249 pendingLine = state.pendingLine;
250 linePosition = state.linePosition;
251 last = state.last;
252 }
253
254 /**
255 * Add a new token to the stream.
256 *
257 * @param type
258 * Token type.
259 * @param start
260 * Start position.
261 * @param end
262 * End position.
263 */
264 protected void add(final TokenType type, final int start, final int end) {
265 // Record last token.
266 last = type;
267
268 // Only emit the last EOL in a cluster.
269 if (type == EOL) {
270 pendingLine = end;
271 linePosition = start;
272 } else {
273 // Write any pending EOL to stream.
274 if (pendingLine != -1) {
275 stream.put(Token.toDesc(EOL, linePosition, pendingLine));
276 pendingLine = -1;
277 }
278
279 // Write token to stream.
280 stream.put(Token.toDesc(type, start, end - start));
281 }
282 }
283
284 /**
285 * Add a new token to the stream.
286 *
287 * @param type
288 * Token type.
289 * @param start
290 * Start position.
291 */
292 protected void add(final TokenType type, final int start) {
293 add(type, start, position);
294 }
295
296 /**
297 * Return the String of valid whitespace characters for regular
298 * expressions in JavaScript
299 * @return regexp whitespace string
300 */
301 public static String getWhitespaceRegExp() {
302 return JAVASCRIPT_WHITESPACE_IN_REGEXP;
303 }
304
305 /**
306 * Skip end of line.
307 *
308 * @param addEOL true if EOL token should be recorded.
309 */
310 private void skipEOL(final boolean addEOL) {
311
312 if (ch0 == '\r') { // detect \r\n pattern
313 skip(1);
314 if (ch0 == '\n') {
315 skip(1);
316 }
317 } else { // all other space, ch0 is guaranteed to be EOL or \0
318 skip(1);
319 }
320
321 // bump up line count
322 line++;
323
324 if (addEOL) {
325 // Add an EOL token.
326 add(EOL, position, line);
327 }
328 }
329
330 /**
331 * Skip over rest of line including end of line.
332 *
333 * @param addEOL true if EOL token should be recorded.
334 */
335 private void skipLine(final boolean addEOL) {
336 // Ignore characters.
337 while (!isEOL(ch0) && !atEOF()) {
338 skip(1);
339 }
340 // Skip over end of line.
341 skipEOL(addEOL);
342 }
343
344 /**
345 * Test whether a char is valid JavaScript whitespace
346 * @param ch a char
347 * @return true if valid JavaScript whitespace
348 */
349 public static boolean isJSWhitespace(final char ch) {
350 return JAVASCRIPT_WHITESPACE.indexOf(ch) != -1;
351 }
352
353 /**
354 * Test whether a char is valid JavaScript end of line
355 * @param ch a char
356 * @return true if valid JavaScript end of line
357 */
358 public static boolean isJSEOL(final char ch) {
359 return JAVASCRIPT_WHITESPACE_EOL.indexOf(ch) != -1;
360 }
361
362 /**
363 * Test whether a char is valid JSON whitespace
364 * @param ch a char
365 * @return true if valid JSON whitespace
366 */
367 public static boolean isJsonWhitespace(final char ch) {
368 return JSON_WHITESPACE.indexOf(ch) != -1;
369 }
370
371 /**
372 * Test whether a char is valid JSON end of line
373 * @param ch a char
374 * @return true if valid JSON end of line
375 */
376 public static boolean isJsonEOL(final char ch) {
377 return JSON_WHITESPACE_EOL.indexOf(ch) != -1;
378 }
379
380 /**
381 * Test if char is a string delimiter, e.g. '\' or '"'. Also scans exec
382 * strings ('`') in scripting mode.
383 * @param ch a char
384 * @return true if string delimiter
385 */
386 protected boolean isStringDelimiter(final char ch) {
387 return ch == '\'' || ch == '"' || (scripting && ch == '`');
388 }
389
390 /**
391 * Test whether a char is valid JavaScript whitespace
392 * @param ch a char
393 * @return true if valid JavaScript whitespace
394 */
395 protected boolean isWhitespace(final char ch) {
396 return Lexer.isJSWhitespace(ch);
397 }
398
399 /**
400 * Test whether a char is valid JavaScript end of line
401 * @param ch a char
402 * @return true if valid JavaScript end of line
403 */
404 protected boolean isEOL(final char ch) {
405 return Lexer.isJSEOL(ch);
406 }
407
408 /**
409 * Skip over whitespace and detect end of line, adding EOL tokens if
410 * encountered.
411 *
412 * @param addEOL true if EOL tokens should be recorded.
413 */
414 private void skipWhitespace(final boolean addEOL) {
415 while (isWhitespace(ch0)) {
416 if (isEOL(ch0)) {
417 skipEOL(addEOL);
418 } else {
419 skip(1);
420 }
421 }
422 }
423
424 /**
425 * Skip over comments.
426 *
427 * @return True if a comment.
428 */
429 protected boolean skipComments() {
430 if (ch0 == '/') {
431 // Is it a // comment.
432 if (ch1 == '/') {
433 // Skip over //.
434 skip(2);
435 // Scan for EOL.
436 while (!atEOF() && !isEOL(ch0)) {
437 skip(1);
438 }
439 // Did detect a comment.
440 return true;
441 } else if (ch1 == '*') {
442 // Record beginning of comment.
443 final int start = position;
444 // Skip over /*.
445 skip(2);
446 // Scan for */.
447 while (!atEOF() && !(ch0 == '*' && ch1 == '/')) {
448 // If end of line handle else skip character.
449 if (isEOL(ch0)) {
450 skipEOL(true);
451 } else {
452 skip(1);
453 }
454 }
455
456 if (atEOF()) {
457 // TODO - Report closing */ missing in parser.
458 add(ERROR, start);
459 } else {
460 // Skip */.
461 skip(2);
462 }
463
464 // Did detect a comment.
465 return true;
466 }
467 }
468
469 if (scripting && ch0 == '#') {
470 // shell style comment
471 // Skip over #.
472 skip(1);
473 // Scan for EOL.
474 while (!atEOF() && !isEOL(ch0)) {
475 skip(1);
476 }
477 // Did detect a comment.
478 return true;
479 }
480
481 // Not a comment.
482 return false;
483 }
484
485 /**
486 * Convert a regex token to a token object.
487 *
488 * @param start Position in source content.
489 * @param length Length of regex token.
490 * @return Regex token object.
491 */
492 public RegexToken valueOfPattern(final int start, final int length) {
493 // Save the current position.
494 final int savePosition = position;
495 // Reset to beginning of content.
496 reset(start);
497 // Buffer for recording characters.
498 final StringBuilder sb = new StringBuilder(length);
499
500 // Skip /.
501 skip(1);
502 boolean inBrackets = false;
503 // Scan for closing /, stopping at end of line.
504 while (!atEOF() && ch0 != '/' && !isEOL(ch0) || inBrackets) {
505 // Skip over escaped character.
506 if (ch0 == '\\') {
507 sb.append(ch0);
508 sb.append(ch1);
509 skip(2);
510 } else {
511 if (ch0 == '[') {
512 inBrackets = true;
513 } else if (ch0 == ']') {
514 inBrackets = false;
515 }
516
517 // Skip literal character.
518 sb.append(ch0);
519 skip(1);
520 }
521 }
522
523 // Get pattern as string.
524 final String regex = sb.toString();
525
526 // Skip /.
527 skip(1);
528
529 // Options as string.
530 final String options = source.getString(position, scanIdentifier());
531
532 reset(savePosition);
533
534 // Compile the pattern.
535 return new RegexToken(regex, options);
536 }
537
538 /**
539 * Return true if the given token can be the beginning of a literal.
540 *
541 * @param token a token
542 * @return true if token can start a literal.
543 */
544 public boolean canStartLiteral(final TokenType token) {
545 return token.startsWith('/') || ((scripting || XML_LITERALS) && token.startsWith('<'));
546 }
547
548 /**
549 * Check whether the given token represents the beginning of a literal. If so scan
550 * the literal and return <tt>true</tt>, otherwise return false.
551 *
552 * @param token the token.
553 * @param startTokenType the token type.
554 * @return True if a literal beginning with startToken was found and scanned.
555 */
556 protected boolean scanLiteral(final long token, final TokenType startTokenType) {
557 // Check if it can be a literal.
558 if (!canStartLiteral(startTokenType)) {
559 return false;
560 }
561 // We break on ambiguous tokens so if we already moved on it can't be a literal.
562 if (stream.get(stream.last()) != token) {
563 return false;
564 }
565 // Rewind to token start position
566 reset(Token.descPosition(token));
567
568 if (ch0 == '/') {
569 return scanRegEx();
570 } else if (ch0 == '<') {
571 if (ch1 == '<') {
572 return scanHereString();
573 } else if (Character.isJavaIdentifierStart(ch1)) {
574 return scanXMLLiteral();
575 }
576 }
577
578 return false;
579 }
580
581 /**
582 * Scan over regex literal.
583 *
584 * @return True if a regex literal.
585 */
586 private boolean scanRegEx() {
587 assert ch0 == '/';
588 // Make sure it's not a comment.
589 if (ch1 != '/' && ch1 != '*') {
590 // Record beginning of literal.
591 final int start = position;
592 // Skip /.
593 skip(1);
594 boolean inBrackets = false;
595
596 // Scan for closing /, stopping at end of line.
597 while (!atEOF() && (ch0 != '/' || inBrackets) && !isEOL(ch0)) {
598 // Skip over escaped character.
599 if (ch0 == '\\') {
600 skip(1);
601 if (isEOL(ch0)) {
602 reset(start);
603 return false;
604 }
605 skip(1);
606 } else {
607 if (ch0 == '[') {
608 inBrackets = true;
609 } else if (ch0 == ']') {
610 inBrackets = false;
611 }
612
613 // Skip literal character.
614 skip(1);
615 }
616 }
617
618 // If regex literal.
619 if (ch0 == '/') {
620 // Skip /.
621 skip(1);
622
623 // Skip over options.
624 while (!atEOF() && Character.isJavaIdentifierPart(ch0) || ch0 == '\\' && ch1 == 'u') {
625 skip(1);
626 }
627
628 // Add regex token.
629 add(REGEX, start);
630 // Regex literal detected.
631 return true;
632 }
633
634 // False start try again.
635 reset(start);
636 }
637
638 // Regex literal not detected.
639 return false;
640 }
641
642 /**
643 * Convert a digit to a integer. Can't use Character.digit since we are
644 * restricted to ASCII by the spec.
645 *
646 * @param ch Character to convert.
647 * @param base Numeric base.
648 *
649 * @return The converted digit or -1 if invalid.
650 */
651 private static int convertDigit(final char ch, final int base) {
652 int digit;
653
654 if ('0' <= ch && ch <= '9') {
655 digit = ch - '0';
656 } else if ('A' <= ch && ch <= 'Z') {
657 digit = ch - 'A' + 10;
658 } else if ('a' <= ch && ch <= 'z') {
659 digit = ch - 'a' + 10;
660 } else {
661 return -1;
662 }
663
664 return digit < base ? digit : -1;
665 }
666
667
668 /**
669 * Get the value of a numeric sequence.
670 *
671 * @param base Numeric base.
672 * @param max Maximum number of digits.
673 * @param skip Skip over escape first.
674 * @param check Tells whether to throw error if a digit is invalid for the given base.
675 * @param type Type of token to report against.
676 *
677 * @return Value of sequence or < 0 if no digits.
678 */
679 private int valueOfSequence(final int base, final int max, final boolean skip, final boolean check, final TokenType type) {
680 assert base == 16 || base == 8 : "base other than 16 or 8";
681 final boolean isHex = base == 16;
682 final int shift = isHex ? 4 : 3;
683 int value = 0;
684
685 if (skip) {
686 skip(2);
687 }
688
689 for (int i = 0; i < max; i++) {
690 final int digit = convertDigit(ch0, base);
691
692 if (digit == -1) {
693 if (check) {
694 error(Lexer.message("invalid." + (isHex ? "hex" : "octal")), type, position, limit);
695 }
696 return i == 0 ? -1 : value;
697 }
698
699 value = value << shift | digit;
700 skip(1);
701 }
702
703 return value;
704 }
705
706 /**
707 * Convert a string to a JavaScript identifier.
708 *
709 * @param start Position in source content.
710 * @param length Length of token.
711 * @return Ident string or null if an error.
712 */
713 private String valueOfIdent(final int start, final int length) throws RuntimeException {
714 // Save the current position.
715 final int savePosition = position;
716 // End of scan.
717 final int end = start + length;
718 // Reset to beginning of content.
719 reset(start);
720 // Buffer for recording characters.
721 final StringBuilder sb = new StringBuilder(length);
722
723 // Scan until end of line or end of file.
724 while (!atEOF() && position < end && !isEOL(ch0)) {
725 // If escape character.
726 if (ch0 == '\\' && ch1 == 'u') {
727 final int ch = valueOfSequence(16, 4, true, true, TokenType.IDENT);
728 if (isWhitespace((char)ch)) {
729 return null;
730 }
731 if (ch < 0) {
732 sb.append('\\');
733 sb.append('u');
734 } else {
735 sb.append((char)ch);
736 }
737 } else {
738 // Add regular character.
739 sb.append(ch0);
740 skip(1);
741 }
742 }
743
744 // Restore position.
745 reset(savePosition);
746
747 return sb.toString();
748 }
749
750 /**
751 * Scan over and identifier or keyword. Handles identifiers containing
752 * encoded Unicode chars.
753 *
754 * Example:
755 *
756 * var \u0042 = 44;
757 */
758 private void scanIdentifierOrKeyword() {
759 // Record beginning of identifier.
760 final int start = position;
761 // Scan identifier.
762 final int length = scanIdentifier();
763 // Check to see if it is a keyword.
764 final TokenType type = TokenLookup.lookupKeyword(content, start, length);
765 // Add keyword or identifier token.
766 add(type, start);
767 }
768
769 /**
770 * Convert a string to a JavaScript string object.
771 *
772 * @param start Position in source content.
773 * @param length Length of token.
774 * @return JavaScript string object.
775 */
776 private String valueOfString(final int start, final int length, final boolean strict) throws RuntimeException {
777 // Save the current position.
778 final int savePosition = position;
779 // Calculate the end position.
780 final int end = start + length;
781 // Reset to beginning of string.
782 reset(start);
783
784 // Buffer for recording characters.
785 final StringBuilder sb = new StringBuilder(length);
786
787 // Scan until end of string.
788 while (position < end) {
789 // If escape character.
790 if (ch0 == '\\') {
791 skip(1);
792
793 final char next = ch0;
794 final int afterSlash = position;
795
796 skip(1);
797
798 // Special characters.
799 switch (next) {
800 case '0':
801 case '1':
802 case '2':
803 case '3':
804 case '4':
805 case '5':
806 case '6':
807 case '7': {
808 if (strict) {
809 // "\0" itself is allowed in strict mode. Only other 'real'
810 // octal escape sequences are not allowed (eg. "\02", "\31").
811 // See section 7.8.4 String literals production EscapeSequence
812 if (next != '0' || (ch0 >= '0' && ch0 <= '9')) {
813 error(Lexer.message("strict.no.octal"), STRING, position, limit);
814 }
815 }
816 reset(afterSlash);
817 // Octal sequence.
818 final int ch = valueOfSequence(8, 3, false, false, STRING);
819
820 if (ch < 0) {
821 sb.append('\\');
822 sb.append('x');
823 } else {
824 sb.append((char)ch);
825 }
826 break;
827 }
828 case 'n':
829 sb.append('\n');
830 break;
831 case 't':
832 sb.append('\t');
833 break;
834 case 'b':
835 sb.append('\b');
836 break;
837 case 'f':
838 sb.append('\f');
839 break;
840 case 'r':
841 sb.append('\r');
842 break;
843 case '\'':
844 sb.append('\'');
845 break;
846 case '\"':
847 sb.append('\"');
848 break;
849 case '\\':
850 sb.append('\\');
851 break;
852 case '\r': // CR | CRLF
853 if (ch0 == '\n') {
854 skip(1);
855 }
856 // fall through
857 case '\n': // LF
858 case '\u2028': // LS
859 case '\u2029': // PS
860 // continue on the next line, slash-return continues string
861 // literal
862 break;
863 case 'x': {
864 // Hex sequence.
865 final int ch = valueOfSequence(16, 2, false, true, STRING);
866
867 if (ch < 0) {
868 sb.append('\\');
869 sb.append('x');
870 } else {
871 sb.append((char)ch);
872 }
873 }
874 break;
875 case 'u': {
876 // Unicode sequence.
877 final int ch = valueOfSequence(16, 4, false, true, STRING);
878
879 if (ch < 0) {
880 sb.append('\\');
881 sb.append('u');
882 } else {
883 sb.append((char)ch);
884 }
885 }
886 break;
887 case 'v':
888 sb.append('\u000B');
889 break;
890 // All other characters.
891 default:
892 sb.append(next);
893 break;
894 }
895 } else {
896 // Add regular character.
897 sb.append(ch0);
898 skip(1);
899 }
900 }
901
902 // Restore position.
903 reset(savePosition);
904
905 return sb.toString();
906 }
907
908 /**
909 * Scan over a string literal.
910 */
911 private void scanString(final boolean add) {
912 // Type of string.
913 TokenType type = STRING;
914 // Record starting quote.
915 final char quote = ch0;
916 // Skip over quote.
917 skip(1);
918
919 // Record beginning of string content.
920 final State stringState = saveState();
921
922 // Scan until close quote or end of line.
923 while (!atEOF() && ch0 != quote && !isEOL(ch0)) {
924 // Skip over escaped character.
925 if (ch0 == '\\') {
926 type = ESCSTRING;
927 skip(1);
928 if (isEOL(ch0)) {
929 // Multiline string literal
930 skipEOL(false);
931 continue;
932 }
933 }
934 // Skip literal character.
935 skip(1);
936 }
937
938 // If close quote.
939 if (ch0 == quote) {
940 // Skip close quote.
941 skip(1);
942 } else {
943 error(Lexer.message("missing.close.quote"), STRING, position, limit);
944 }
945
946 // If not just scanning.
947 if (add) {
948 // Record end of string.
949 stringState.setLimit(position - 1);
950
951 if (scripting && !stringState.isEmpty()) {
952 switch (quote) {
953 case '`':
954 // Mark the beginning of an exec string.
955 add(EXECSTRING, stringState.position, stringState.limit);
956 // Frame edit string with left brace.
957 add(LBRACE, stringState.position, stringState.position);
958 // Process edit string.
959 editString(type, stringState);
960 // Frame edit string with right brace.
961 add(RBRACE, stringState.limit, stringState.limit);
962 break;
963 case '"':
964 // Only edit double quoted strings.
965 editString(type, stringState);
966 break;
967 case '\'':
968 // Add string token without editing.
969 add(type, stringState.position, stringState.limit);
970 break;
971 default:
972 break;
973 }
974 } else {
975 /// Add string token without editing.
976 add(type, stringState.position, stringState.limit);
977 }
978 }
979 }
980
981 /**
982 * Convert string to number.
983 *
984 * @param valueString String to convert.
985 * @param radix Numeric base.
986 * @return Converted number.
987 */
988 private static Number valueOf(final String valueString, final int radix) throws NumberFormatException {
989 try {
990 final long value = Long.parseLong(valueString, radix);
991 if(value >= MIN_INT_L && value <= MAX_INT_L) {
992 return Integer.valueOf((int)value);
993 }
994 return Long.valueOf(value);
995 } catch (final NumberFormatException e) {
996 if (radix == 10) {
997 return Double.valueOf(valueString);
998 }
999
1000 double value = 0.0;
1001
1002 for (int i = 0; i < valueString.length(); i++) {
1003 final char ch = valueString.charAt(i);
1004 // Preverified, should always be a valid digit.
1005 final int digit = convertDigit(ch, radix);
1006 value *= radix;
1007 value += digit;
1008 }
1009
1010 return value;
1011 }
1012 }
1013
1014 /**
1015 * Convert string to number.
1016 *
1017 * @param valueString String to convert.
1018 * @return Converted number.
1019 */
1020 private static Number valueOf(final String valueString) throws NumberFormatException {
1021 return JSType.narrowestIntegerRepresentation(Double.valueOf(valueString));
1022 }
1023
1024 /**
1025 * Scan a number.
1026 */
1027 private void scanNumber() {
1028 // Record beginning of number.
1029 final int start = position;
1030 // Assume value is a decimal.
1031 TokenType type = DECIMAL;
1032
1033 // First digit of number.
1034 int digit = convertDigit(ch0, 10);
1035
1036 // If number begins with 0x.
1037 if (digit == 0 && (ch1 == 'x' || ch1 == 'X') && convertDigit(ch2, 16) != -1) {
1038 // Skip over 0xN.
1039 skip(3);
1040 // Skip over remaining digits.
1041 while (convertDigit(ch0, 16) != -1) {
1042 skip(1);
1043 }
1044
1045 type = HEXADECIMAL;
1046 } else {
1047 // Check for possible octal constant.
1048 boolean octal = digit == 0;
1049 // Skip first digit if not leading '.'.
1050 if (digit != -1) {
1051 skip(1);
1052 }
1053
1054 // Skip remaining digits.
1055 while ((digit = convertDigit(ch0, 10)) != -1) {
1056 // Check octal only digits.
1057 octal = octal && digit < 8;
1058 // Skip digit.
1059 skip(1);
1060 }
1061
1062 if (octal && position - start > 1) {
1063 type = OCTAL;
1064 } else if (ch0 == '.' || ch0 == 'E' || ch0 == 'e') {
1065 // Must be a double.
1066 if (ch0 == '.') {
1067 // Skip period.
1068 skip(1);
1069 // Skip mantissa.
1070 while (convertDigit(ch0, 10) != -1) {
1071 skip(1);
1072 }
1073 }
1074
1075 // Detect exponent.
1076 if (ch0 == 'E' || ch0 == 'e') {
1077 // Skip E.
1078 skip(1);
1079 // Detect and skip exponent sign.
1080 if (ch0 == '+' || ch0 == '-') {
1081 skip(1);
1082 }
1083 // Skip exponent.
1084 while (convertDigit(ch0, 10) != -1) {
1085 skip(1);
1086 }
1087 }
1088
1089 type = FLOATING;
1090 }
1091 }
1092
1093 // Add number token.
1094 add(type, start);
1095 }
1096
1097 /**
1098 * Convert a regex token to a token object.
1099 *
1100 * @param start Position in source content.
1101 * @param length Length of regex token.
1102 * @return Regex token object.
1103 */
1104 XMLToken valueOfXML(final int start, final int length) {
1105 return new XMLToken(source.getString(start, length));
1106 }
1107
1108 /**
1109 * Scan over a XML token.
1110 *
1111 * @return TRUE if is an XML literal.
1112 */
1113 private boolean scanXMLLiteral() {
1114 assert ch0 == '<' && Character.isJavaIdentifierStart(ch1);
1115 if (XML_LITERALS) {
1116 // Record beginning of xml expression.
1117 final int start = position;
1118
1119 int openCount = 0;
1120
1121 do {
1122 if (ch0 == '<') {
1123 if (ch1 == '/' && Character.isJavaIdentifierStart(ch2)) {
1124 skip(3);
1125 openCount--;
1126 } else if (Character.isJavaIdentifierStart(ch1)) {
1127 skip(2);
1128 openCount++;
1129 } else if (ch1 == '?') {
1130 skip(2);
1131 } else if (ch1 == '!' && ch2 == '-' && ch3 == '-') {
1132 skip(4);
1133 } else {
1134 reset(start);
1135 return false;
1136 }
1137
1138 while (!atEOF() && ch0 != '>') {
1139 if (ch0 == '/' && ch1 == '>') {
1140 openCount--;
1141 skip(1);
1142 break;
1143 } else if (ch0 == '\"' || ch0 == '\'') {
1144 scanString(false);
1145 } else {
1146 skip(1);
1147 }
1148 }
1149
1150 if (ch0 != '>') {
1151 reset(start);
1152 return false;
1153 }
1154
1155 skip(1);
1156 } else if (atEOF()) {
1157 reset(start);
1158 return false;
1159 } else {
1160 skip(1);
1161 }
1162 } while (openCount > 0);
1163
1164 add(XML, start);
1165 return true;
1166 }
1167
1168 return false;
1169 }
1170
1171 /**
1172 * Scan over identifier characters.
1173 *
1174 * @return Length of identifier or zero if none found.
1175 */
1176 private int scanIdentifier() {
1177 final int start = position;
1178
1179 // Make sure first character is valid start character.
1180 if (ch0 == '\\' && ch1 == 'u') {
1181 final int ch = valueOfSequence(16, 4, true, true, TokenType.IDENT);
1182
1183 if (!Character.isJavaIdentifierStart(ch)) {
1184 error(Lexer.message("illegal.identifier.character"), TokenType.IDENT, start, position);
1185 }
1186 } else if (!Character.isJavaIdentifierStart(ch0)) {
1187 // Not an identifier.
1188 return 0;
1189 }
1190
1191 // Make sure remaining characters are valid part characters.
1192 while (!atEOF()) {
1193 if (ch0 == '\\' && ch1 == 'u') {
1194 final int ch = valueOfSequence(16, 4, true, true, TokenType.IDENT);
1195
1196 if (!Character.isJavaIdentifierPart(ch)) {
1197 error(Lexer.message("illegal.identifier.character"), TokenType.IDENT, start, position);
1198 }
1199 } else if (Character.isJavaIdentifierPart(ch0)) {
1200 skip(1);
1201 } else {
1202 break;
1203 }
1204 }
1205
1206 // Length of identifier sequence.
1207 return position - start;
1208 }
1209
1210 /**
1211 * Compare two identifiers (in content) for equality.
1212 *
1213 * @param aStart Start of first identifier.
1214 * @param aLength Length of first identifier.
1215 * @param bStart Start of second identifier.
1216 * @param bLength Length of second identifier.
1217 * @return True if equal.
1218 */
1219 private boolean identifierEqual(final int aStart, final int aLength, final int bStart, final int bLength) {
1220 if (aLength == bLength) {
1221 for (int i = 0; i < aLength; i++) {
1222 if (content[aStart + i] != content[bStart + i]) {
1223 return false;
1224 }
1225 }
1226
1227 return true;
1228 }
1229
1230 return false;
1231 }
1232
1233 /**
1234 * Detect if a line starts with a marker identifier.
1235 *
1236 * @param identStart Start of identifier.
1237 * @param identLength Length of identifier.
1238 * @return True if detected.
1239 */
1240 private boolean hasHereMarker(final int identStart, final int identLength) {
1241 // Skip any whitespace.
1242 skipWhitespace(false);
1243
1244 return identifierEqual(identStart, identLength, position, scanIdentifier());
1245 }
1246
1247 /**
1248 * Lexer to service edit strings.
1249 */
1250 private static class EditStringLexer extends Lexer {
1251 /** Type of string literals to emit. */
1252 final TokenType stringType;
1253
1254 /*
1255 * Constructor.
1256 */
1257
1258 EditStringLexer(final Lexer lexer, final TokenType stringType, final State stringState) {
1259 super(lexer, stringState);
1260
1261 this.stringType = stringType;
1262 }
1263
1264 /**
1265 * Lexify the contents of the string.
1266 */
1267 @Override
1268 public void lexify() {
1269 // Record start of string position.
1270 int stringStart = position;
1271 // Indicate that the priming first string has not been emitted.
1272 boolean primed = false;
1273
1274 while (true) {
1275 // Detect end of content.
1276 if (atEOF()) {
1277 break;
1278 }
1279
1280 // Honour escapes (should be well formed.)
1281 if (ch0 == '\\' && stringType == ESCSTRING) {
1282 skip(2);
1283
1284 continue;
1285 }
1286
1287 // If start of expression.
1288 if (ch0 == '$' && ch1 == '{') {
1289 if (!primed || stringStart != position) {
1290 if (primed) {
1291 add(ADD, stringStart, stringStart + 1);
1292 }
1293
1294 add(stringType, stringStart, position);
1295 primed = true;
1296 }
1297
1298 // Skip ${
1299 skip(2);
1300
1301 // Save expression state.
1302 final State expressionState = saveState();
1303
1304 // Start with one open brace.
1305 int braceCount = 1;
1306
1307 // Scan for the rest of the string.
1308 while (!atEOF()) {
1309 // If closing brace.
1310 if (ch0 == '}') {
1311 // Break only only if matching brace.
1312 if (--braceCount == 0) {
1313 break;
1314 }
1315 } else if (ch0 == '{') {
1316 // Bump up the brace count.
1317 braceCount++;
1318 }
1319
1320 // Skip to next character.
1321 skip(1);
1322 }
1323
1324 // If braces don't match then report an error.
1325 if (braceCount != 0) {
1326 error(Lexer.message("edit.string.missing.brace"), LBRACE, expressionState.position - 1, 1);
1327 }
1328
1329 // Mark end of expression.
1330 expressionState.setLimit(position);
1331 // Skip closing brace.
1332 skip(1);
1333
1334 // Start next string.
1335 stringStart = position;
1336
1337 // Concatenate expression.
1338 add(ADD, expressionState.position, expressionState.position + 1);
1339 add(LPAREN, expressionState.position, expressionState.position + 1);
1340
1341 // Scan expression.
1342 final Lexer lexer = new Lexer(this, expressionState);
1343 lexer.lexify();
1344
1345 // Close out expression parenthesis.
1346 add(RPAREN, position - 1, position);
1347
1348 continue;
1349 }
1350
1351 // Next character in string.
1352 skip(1);
1353 }
1354
1355 // If there is any unemitted string portion.
1356 if (stringStart != limit) {
1357 // Concatenate remaining string.
1358 if (primed) {
1359 add(ADD, stringStart, 1);
1360 }
1361
1362 add(stringType, stringStart, limit);
1363 }
1364 }
1365
1366 }
1367
1368 /**
1369 * Edit string for nested expressions.
1370 *
1371 * @param stringType Type of string literals to emit.
1372 * @param stringState State of lexer at start of string.
1373 */
1374 private void editString(final TokenType stringType, final State stringState) {
1375 // Use special lexer to scan string.
1376 final EditStringLexer lexer = new EditStringLexer(this, stringType, stringState);
1377 lexer.lexify();
1378
1379 // Need to keep lexer informed.
1380 last = stringType;
1381 }
1382
1383 /**
1384 * Scan over a here string.
1385 *
1386 * @return TRUE if is a here string.
1387 */
1388 private boolean scanHereString() {
1389 assert ch0 == '<' && ch1 == '<';
1390 if (scripting) {
1391 // Record beginning of here string.
1392 final State saved = saveState();
1393
1394 // << or <<<
1395 final boolean excludeLastEOL = ch2 != '<';
1396
1397 if (excludeLastEOL) {
1398 skip(2);
1399 } else {
1400 skip(3);
1401 }
1402
1403 // Scan identifier.
1404 final int identStart = position;
1405 final int identLength = scanIdentifier();
1406
1407 // Check for identifier.
1408 if (identLength == 0) {
1409 // Treat as shift.
1410 restoreState(saved);
1411
1412 return false;
1413 }
1414
1415 // Record rest of line.
1416 final State restState = saveState();
1417 skipLine(false);
1418 restState.setLimit(position);
1419
1420 // Record beginning of string.
1421 final State stringState = saveState();
1422 int stringEnd = position;
1423
1424 // Hunt down marker.
1425 while (!atEOF()) {
1426 // Skip any whitespace.
1427 skipWhitespace(false);
1428
1429 if (hasHereMarker(identStart, identLength)) {
1430 break;
1431 }
1432
1433 skipLine(false);
1434 stringEnd = position;
1435 }
1436
1437 // Record end of string.
1438 stringState.setLimit(stringEnd);
1439
1440 // If marker is missing.
1441 if (stringState.isEmpty() || atEOF()) {
1442 error(Lexer.message("here.missing.end.marker", source.getString(identStart, identLength)), last, position, position);
1443 restoreState(saved);
1444
1445 return false;
1446 }
1447
1448 // Remove last end of line if specified.
1449 if (excludeLastEOL) {
1450 // Handles \n.
1451 if (content[stringEnd - 1] == '\n') {
1452 stringEnd--;
1453 }
1454
1455 // Handles \r and \r\n.
1456 if (content[stringEnd - 1] == '\r') {
1457 stringEnd--;
1458 }
1459
1460 // Update end of string.
1461 stringState.setLimit(stringEnd);
1462 }
1463
1464 // Edit string if appropriate.
1465 if (scripting && !stringState.isEmpty()) {
1466 editString(STRING, stringState);
1467 } else {
1468 // Add here string.
1469 add(STRING, stringState.position, stringState.limit);
1470 }
1471
1472 // Scan rest of original line.
1473 final Lexer restLexer = new Lexer(this, restState);
1474
1475 restLexer.lexify();
1476
1477 return true;
1478 }
1479
1480 return false;
1481 }
1482
1483 /**
1484 * Breaks source content down into lex units, adding tokens to the token
1485 * stream. The routine scans until the stream buffer is full. Can be called
1486 * repeatedly until EOF is detected.
1487 */
1488 public void lexify() {
1489 while (!stream.isFull() || nested) {
1490 // Skip over whitespace.
1491 skipWhitespace(true);
1492
1493 // Detect end of file.
1494 if (atEOF()) {
1495 if (!nested) {
1496 // Add an EOF token at the end.
1497 add(EOF, position);
1498 }
1499
1500 break;
1501 }
1502
1503 // Check for comments. Note that we don't scan for regexp and other literals here as
1504 // we may not have enough context to distinguish them from similar looking operators.
1505 // Instead we break on ambiguous operators below and let the parser decide.
1506 if (ch0 == '/' && skipComments()) {
1507 continue;
1508 }
1509
1510 if (scripting && ch0 == '#' && skipComments()) {
1511 continue;
1512 }
1513
1514 // TokenType for lookup of delimiter or operator.
1515 TokenType type;
1516
1517 if (ch0 == '.' && convertDigit(ch1, 10) != -1) {
1518 // '.' followed by digit.
1519 // Scan and add a number.
1520 scanNumber();
1521 } else if ((type = TokenLookup.lookupOperator(ch0, ch1, ch2, ch3)) != null) {
1522 // Get the number of characters in the token.
1523 final int typeLength = type.getLength();
1524 // Skip that many characters.
1525 skip(typeLength);
1526 // Add operator token.
1527 add(type, position - typeLength);
1528 // Some operator tokens also mark the beginning of regexp, XML, or here string literals.
1529 // We break to let the parser decide what it is.
1530 if (canStartLiteral(type)) {
1531 break;
1532 }
1533 } else if (Character.isJavaIdentifierStart(ch0) || ch0 == '\\' && ch1 == 'u') {
1534 // Scan and add identifier or keyword.
1535 scanIdentifierOrKeyword();
1536 } else if (isStringDelimiter(ch0)) {
1537 // Scan and add a string.
1538 scanString(true);
1539 } else if (Character.isDigit(ch0)) {
1540 // Scan and add a number.
1541 scanNumber();
1542 } else {
1543 // Don't recognize this character.
1544 skip(1);
1545 add(ERROR, position - 1);
1546 }
1547 }
1548 }
1549
1550 /**
1551 * Return value of token given its token descriptor.
1552 *
1553 * @param token Token descriptor.
1554 * @return JavaScript value.
1555 */
1556 Object getValueOf(final long token, final boolean strict) {
1557 final int start = Token.descPosition(token);
1558 final int len = Token.descLength(token);
1559
1560 switch (Token.descType(token)) {
1561 case DECIMAL:
1562 return Lexer.valueOf(source.getString(start, len), 10); // number
1563 case OCTAL:
1564 return Lexer.valueOf(source.getString(start, len), 8); // number
1565 case HEXADECIMAL:
1566 return Lexer.valueOf(source.getString(start + 2, len - 2), 16); // number
1567 case FLOATING:
1568 return Lexer.valueOf(source.getString(start, len)); // number
1569 case STRING:
1570 return source.getString(start, len); // String
1571 case ESCSTRING:
1572 return valueOfString(start, len, strict); // String
1573 case IDENT:
1574 return valueOfIdent(start, len); // String
1575 case REGEX:
1576 return valueOfPattern(start, len); // RegexToken::LexerToken
1577 case XML:
1578 return valueOfXML(start, len); // XMLToken::LexerToken
1579 default:
1580 break;
1581 }
1582
1583 return null;
1584 }
1585
1586 private static String message(final String msgId, final String... args) {
1587 return ECMAErrors.getMessage("lexer.error." + msgId, args);
1588 }
1589
1590 /**
1591 * Generate a runtime exception
1592 *
1593 * @param message error message
1594 * @param type token type
1595 * @param start start position of lexed error
1596 * @param length length of lexed error
1597 * @throws ParserException unconditionally
1598 */
1599 protected void error(final String message, final TokenType type, final int start, final int length) throws ParserException {
1600 final long token = Token.toDesc(type, start, length);
1601 final int pos = Token.descPosition(token);
1602 final int lineNum = source.getLine(pos);
1603 final int columnNum = source.getColumn(pos);
1604 final String formatted = ErrorManager.format(message, source, lineNum, columnNum, token);
1605 throw new ParserException(JSErrorType.SYNTAX_ERROR, formatted, source, lineNum, columnNum, token);
1606 }
1607
1608 /**
1609 * Helper class for Lexer tokens, e.g XML or RegExp tokens.
1610 * This is the abstract superclass
1611 */
1612 public static abstract class LexerToken {
1613 private final String expression;
1614
1615 /**
1616 * Constructor
1617 * @param expression token expression
1618 */
1619 protected LexerToken(final String expression) {
1620 this.expression = expression;
1621 }
1622
1623 /**
1624 * Get the expression
1625 * @return expression
1626 */
1627 public String getExpression() {
1628 return expression;
1629 }
1630 }
1631
1632 /**
1633 * Temporary container for regular expressions.
1634 */
1635 public static class RegexToken extends LexerToken {
1636 /** Options. */
1637 private final String options;
1638
1639 /**
1640 * Constructor.
1641 *
1642 * @param expression regexp expression
1643 * @param options regexp options
1644 */
1645 public RegexToken(final String expression, final String options) {
1646 super(expression);
1647 this.options = options;
1648 }
1649
1650 /**
1651 * Get regexp options
1652 * @return options
1653 */
1654 public String getOptions() {
1655 return options;
1656 }
1657
1658 @Override
1659 public String toString() {
1660 return '/' + getExpression() + '/' + options;
1661 }
1662 }
1663
1664 /**
1665 * Temporary container for XML expression.
1666 */
1667 public static class XMLToken extends LexerToken {
1668
1669 /**
1670 * Constructor.
1671 *
1672 * @param expression XML expression
1673 */
1674 public XMLToken(final String expression) {
1675 super(expression);
1676 }
1677 }
1678 }