1 /*
2 * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.nashorn.internal.parser;
27
28 import static jdk.nashorn.internal.parser.TokenKind.LITERAL;
29
30 import jdk.nashorn.internal.runtime.Source;
31
32 /**
33 * A token is a 64 bit long value that represents a basic parse/lex unit.
34 * This class provides static methods to manipulate lexer tokens.
35 */
36 public class Token {
37
38 /**
39 * We use 28 bits for the position and 28 bits for the length of the token.
40 * This limits the maximal length of code we can handle to 2 ^ 28 - 1 bytes.
41 */
42 public final static int LENGTH_MASK = 0xfffffff;
43
44 // The first 8 bits are used for the token type, followed by length and position
45 private final static int LENGTH_SHIFT = 8;
46 private final static int POSITION_SHIFT = 36;
47
48 private Token() {
49 }
50
51 /**
52 * Create a compact form of token information.
53 * @param type Type of token.
54 * @param position Start position of the token in the source.
55 * @param length Length of the token.
56 * @return Token descriptor.
57 */
58 public static long toDesc(final TokenType type, final int position, final int length) {
59 assert position <= LENGTH_MASK && length <= LENGTH_MASK;
60 return (long)position << POSITION_SHIFT |
61 (long)length << LENGTH_SHIFT |
62 type.ordinal();
63 }
64
65 /**
66 * Extract token position from a token descriptor.
67 * @param token Token descriptor.
68 * @return Start position of the token in the source.
69 */
70 public static int descPosition(final long token) {
71 return (int)(token >>> POSITION_SHIFT);
72 }
73
74 /**
75 * Normally returns the token itself, except in case of string tokens
76 * which report their position past their opening delimiter and thus
77 * need to have position and length adjusted.
78 *
79 * @param token Token descriptor.
80 * @return same or adjusted token.
81 */
82 public static long withDelimiter(final long token) {
83 final TokenType tokenType = Token.descType(token);
84 switch(tokenType) {
85 case STRING:
86 case ESCSTRING:
87 case EXECSTRING:
88 case TEMPLATE:
89 case TEMPLATE_TAIL: {
90 final int start = Token.descPosition(token) - 1;
91 final int len = Token.descLength(token) + 2;
92 return toDesc(tokenType, start, len);
93 }
94 case TEMPLATE_HEAD:
95 case TEMPLATE_MIDDLE: {
96 final int start = Token.descPosition(token) - 1;
97 final int len = Token.descLength(token) + 3;
98 return toDesc(tokenType, start, len);
99 }
100 default: {
101 return token;
102 }
103 }
104 }
105
106 /**
107 * Extract token length from a token descriptor.
108 * @param token Token descriptor.
109 * @return Length of the token.
110 */
111 public static int descLength(final long token) {
112 return (int)((token >>> LENGTH_SHIFT) & LENGTH_MASK);
113 }
114
115 /**
116 * Extract token type from a token descriptor.
117 * @param token Token descriptor.
118 * @return Type of token.
119 */
120 public static TokenType descType(final long token) {
121 return TokenType.getValues()[(int)token & 0xff];
122 }
123
124 /**
125 * Change the token to use a new type.
126 *
127 * @param token The original token.
128 * @param newType The new token type.
129 * @return The recast token.
130 */
131 public static long recast(final long token, final TokenType newType) {
132 return token & ~0xFFL | newType.ordinal();
133 }
134
135 /**
136 * Return a string representation of a token.
137 * @param source Token source.
138 * @param token Token descriptor.
139 * @param verbose True to include details.
140 * @return String representation.
141 */
142 public static String toString(final Source source, final long token, final boolean verbose) {
143 final TokenType type = Token.descType(token);
144 String result;
145
146 if (source != null && type.getKind() == LITERAL) {
147 result = source.getString(token);
148 } else {
149 result = type.getNameOrType();
150 }
151
152 if (verbose) {
153 final int position = Token.descPosition(token);
154 final int length = Token.descLength(token);
155 result += " (" + position + ", " + length + ")";
156 }
157
158 return result;
159 }
160
161 /**
162 * String conversion of token
163 *
164 * @param source the source
165 * @param token the token
166 *
167 * @return token as string
168 */
169 public static String toString(final Source source, final long token) {
170 return Token.toString(source, token, false);
171 }
172
173 /**
174 * String conversion of token - version without source given
175 *
176 * @param token the token
177 *
178 * @return token as string
179 */
180 public static String toString(final long token) {
181 return Token.toString(null, token, false);
182 }
183
184 /**
185 * Static hash code computation function token
186 *
187 * @param token a token
188 *
189 * @return hash code for token
190 */
191 public static int hashCode(final long token) {
192 return (int)(token ^ token >>> 32);
193 }
194
195 }