44 * @param position Start position of the token in the source. 45 * @param length Length of the token. 46 * @return Token descriptor. 47 */ 48 public static long toDesc(final TokenType type, final int position, final int length) { 49 return (long)position << 32 | 50 (long)length << 8 | 51 type.ordinal(); 52 } 53 54 /** 55 * Extract token position from a token descriptor. 56 * @param token Token descriptor. 57 * @return Start position of the token in the source. 58 */ 59 public static int descPosition(final long token) { 60 return (int)(token >>> 32); 61 } 62 63 /** 64 * Extract token length from a token descriptor. 65 * @param token Token descriptor. 66 * @return Length of the token. 67 */ 68 public static int descLength(final long token) { 69 return (int)token >>> 8; 70 } 71 72 /** 73 * Extract token type from a token descriptor. 74 * @param token Token descriptor. 75 * @return Type of token. 76 */ 77 public static TokenType descType(final long token) { 78 return TokenType.getValues()[(int)token & 0xff]; 79 } 80 81 /** 82 * Change the token to use a new type. 83 * | 44 * @param position Start position of the token in the source. 45 * @param length Length of the token. 46 * @return Token descriptor. 47 */ 48 public static long toDesc(final TokenType type, final int position, final int length) { 49 return (long)position << 32 | 50 (long)length << 8 | 51 type.ordinal(); 52 } 53 54 /** 55 * Extract token position from a token descriptor. 56 * @param token Token descriptor. 57 * @return Start position of the token in the source. 58 */ 59 public static int descPosition(final long token) { 60 return (int)(token >>> 32); 61 } 62 63 /** 64 * Normally returns the token itself, except in case of string tokens 65 * which report their position past their opening delimiter and thus 66 * need to have position and length adjusted. 67 * 68 * @param token Token descriptor. 69 * @return same or adjusted token. 70 */ 71 public static long withDelimiter(final long token) { 72 final TokenType tokenType = Token.descType(token); 73 switch(tokenType) { 74 case STRING: case ESCSTRING: case EXECSTRING: { 75 final int start = Token.descPosition(token) - 1; 76 final int len = Token.descLength(token) + 2; 77 return toDesc(tokenType, start, len); 78 } 79 default: { 80 return token; 81 } 82 } 83 } 84 85 /** 86 * Extract token length from a token descriptor. 87 * @param token Token descriptor. 88 * @return Length of the token. 89 */ 90 public static int descLength(final long token) { 91 return (int)token >>> 8; 92 } 93 94 /** 95 * Extract token type from a token descriptor. 96 * @param token Token descriptor. 97 * @return Type of token. 98 */ 99 public static TokenType descType(final long token) { 100 return TokenType.getValues()[(int)token & 0xff]; 101 } 102 103 /** 104 * Change the token to use a new type. 105 * |