51 * <li>Whether the characters of identifiers are converted to lowercase.
52 * </ul>
53 * <p>
54 * A typical application first constructs an instance of this class,
55 * sets up the syntax tables, and then repeatedly loops calling the
56 * {@code nextToken} method in each iteration of the loop until
57 * it returns the value {@code TT_EOF}.
58 *
59 * @author James Gosling
60 * @see java.io.StreamTokenizer#nextToken()
61 * @see java.io.StreamTokenizer#TT_EOF
62 * @since 1.0
63 */
64
65 public class StreamTokenizer {
66
67 /* Only one of these will be non-null */
68 private Reader reader = null;
69 private InputStream input = null;
70
71 private char buf[] = new char[20];
72
73 /**
74 * The next character to be considered by the nextToken method. May also
75 * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
76 * to indicate that a new character should be read and, if it is a '\n'
77 * character, it should be discarded and a second new character should be
78 * read.
79 */
80 private int peekc = NEED_CHAR;
81
82 private static final int NEED_CHAR = Integer.MAX_VALUE;
83 private static final int SKIP_LF = Integer.MAX_VALUE - 1;
84
85 private boolean pushedBack;
86 private boolean forceLower;
87 /** The line number of the last token read */
88 private int LINENO = 1;
89
90 private boolean eolIsSignificantP = false;
91 private boolean slashSlashCommentsP = false;
92 private boolean slashStarCommentsP = false;
93
94 private byte ctype[] = new byte[256];
95 private static final byte CT_WHITESPACE = 1;
96 private static final byte CT_DIGIT = 2;
97 private static final byte CT_ALPHA = 4;
98 private static final byte CT_QUOTE = 8;
99 private static final byte CT_COMMENT = 16;
100
101 /**
102 * After a call to the {@code nextToken} method, this field
103 * contains the type of the token just read. For a single character
104 * token, its value is the single character, converted to an integer.
105 * For a quoted string token, its value is the quote character.
106 * Otherwise, its value is one of the following:
107 * <ul>
108 * <li>{@code TT_WORD} indicates that the token is a word.
109 * <li>{@code TT_NUMBER} indicates that the token is a number.
110 * <li>{@code TT_EOL} indicates that the end of line has been read.
111 * The field can only have this value if the
112 * {@code eolIsSignificant} method has been called with the
113 * argument {@code true}.
114 * <li>{@code TT_EOF} indicates that the end of the input stream
510 * field. Additional information about the token may be in the
511 * {@code nval} field or the {@code sval} field of this
512 * tokenizer.
513 * <p>
514 * Typical clients of this
515 * class first set up the syntax tables and then sit in a loop
516 * calling nextToken to parse successive tokens until TT_EOF
517 * is returned.
518 *
519 * @return the value of the {@code ttype} field.
520 * @exception IOException if an I/O error occurs.
521 * @see java.io.StreamTokenizer#nval
522 * @see java.io.StreamTokenizer#sval
523 * @see java.io.StreamTokenizer#ttype
524 */
525 public int nextToken() throws IOException {
526 if (pushedBack) {
527 pushedBack = false;
528 return ttype;
529 }
530 byte ct[] = ctype;
531 sval = null;
532
533 int c = peekc;
534 if (c < 0)
535 c = NEED_CHAR;
536 if (c == SKIP_LF) {
537 c = read();
538 if (c < 0)
539 return ttype = TT_EOF;
540 if (c == '\n')
541 c = NEED_CHAR;
542 }
543 if (c == NEED_CHAR) {
544 c = read();
545 if (c < 0)
546 return ttype = TT_EOF;
547 }
548 ttype = c; /* Just to be safe */
549
550 /* Set peekc so that the next invocation of nextToken will read
804 break;
805 case TT_NUMBER:
806 ret = "n=" + nval;
807 break;
808 case TT_NOTHING:
809 ret = "NOTHING";
810 break;
811 default: {
812 /*
813 * ttype is the first character of either a quoted string or
814 * is an ordinary character. ttype can definitely not be less
815 * than 0, since those are reserved values used in the previous
816 * case statements
817 */
818 if (ttype < 256 &&
819 ((ctype[ttype] & CT_QUOTE) != 0)) {
820 ret = sval;
821 break;
822 }
823
824 char s[] = new char[3];
825 s[0] = s[2] = '\'';
826 s[1] = (char) ttype;
827 ret = new String(s);
828 break;
829 }
830 }
831 return "Token[" + ret + "], line " + LINENO;
832 }
833
834 }
|
51 * <li>Whether the characters of identifiers are converted to lowercase.
52 * </ul>
53 * <p>
54 * A typical application first constructs an instance of this class,
55 * sets up the syntax tables, and then repeatedly loops calling the
56 * {@code nextToken} method in each iteration of the loop until
57 * it returns the value {@code TT_EOF}.
58 *
59 * @author James Gosling
60 * @see java.io.StreamTokenizer#nextToken()
61 * @see java.io.StreamTokenizer#TT_EOF
62 * @since 1.0
63 */
64
65 public class StreamTokenizer {
66
67 /* Only one of these will be non-null */
68 private Reader reader = null;
69 private InputStream input = null;
70
71 private char[] buf = new char[20];
72
73 /**
74 * The next character to be considered by the nextToken method. May also
75 * be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
76 * to indicate that a new character should be read and, if it is a '\n'
77 * character, it should be discarded and a second new character should be
78 * read.
79 */
80 private int peekc = NEED_CHAR;
81
82 private static final int NEED_CHAR = Integer.MAX_VALUE;
83 private static final int SKIP_LF = Integer.MAX_VALUE - 1;
84
85 private boolean pushedBack;
86 private boolean forceLower;
87 /** The line number of the last token read */
88 private int LINENO = 1;
89
90 private boolean eolIsSignificantP = false;
91 private boolean slashSlashCommentsP = false;
92 private boolean slashStarCommentsP = false;
93
94 private byte[] ctype = new byte[256];
95 private static final byte CT_WHITESPACE = 1;
96 private static final byte CT_DIGIT = 2;
97 private static final byte CT_ALPHA = 4;
98 private static final byte CT_QUOTE = 8;
99 private static final byte CT_COMMENT = 16;
100
101 /**
102 * After a call to the {@code nextToken} method, this field
103 * contains the type of the token just read. For a single character
104 * token, its value is the single character, converted to an integer.
105 * For a quoted string token, its value is the quote character.
106 * Otherwise, its value is one of the following:
107 * <ul>
108 * <li>{@code TT_WORD} indicates that the token is a word.
109 * <li>{@code TT_NUMBER} indicates that the token is a number.
110 * <li>{@code TT_EOL} indicates that the end of line has been read.
111 * The field can only have this value if the
112 * {@code eolIsSignificant} method has been called with the
113 * argument {@code true}.
114 * <li>{@code TT_EOF} indicates that the end of the input stream
510 * field. Additional information about the token may be in the
511 * {@code nval} field or the {@code sval} field of this
512 * tokenizer.
513 * <p>
514 * Typical clients of this
515 * class first set up the syntax tables and then sit in a loop
516 * calling nextToken to parse successive tokens until TT_EOF
517 * is returned.
518 *
519 * @return the value of the {@code ttype} field.
520 * @exception IOException if an I/O error occurs.
521 * @see java.io.StreamTokenizer#nval
522 * @see java.io.StreamTokenizer#sval
523 * @see java.io.StreamTokenizer#ttype
524 */
525 public int nextToken() throws IOException {
526 if (pushedBack) {
527 pushedBack = false;
528 return ttype;
529 }
530 byte[] ct = ctype;
531 sval = null;
532
533 int c = peekc;
534 if (c < 0)
535 c = NEED_CHAR;
536 if (c == SKIP_LF) {
537 c = read();
538 if (c < 0)
539 return ttype = TT_EOF;
540 if (c == '\n')
541 c = NEED_CHAR;
542 }
543 if (c == NEED_CHAR) {
544 c = read();
545 if (c < 0)
546 return ttype = TT_EOF;
547 }
548 ttype = c; /* Just to be safe */
549
550 /* Set peekc so that the next invocation of nextToken will read
804 break;
805 case TT_NUMBER:
806 ret = "n=" + nval;
807 break;
808 case TT_NOTHING:
809 ret = "NOTHING";
810 break;
811 default: {
812 /*
813 * ttype is the first character of either a quoted string or
814 * is an ordinary character. ttype can definitely not be less
815 * than 0, since those are reserved values used in the previous
816 * case statements
817 */
818 if (ttype < 256 &&
819 ((ctype[ttype] & CT_QUOTE) != 0)) {
820 ret = sval;
821 break;
822 }
823
824 char[] s = new char[3];
825 s[0] = s[2] = '\'';
826 s[1] = (char) ttype;
827 ret = new String(s);
828 break;
829 }
830 }
831 return "Token[" + ret + "], line " + LINENO;
832 }
833
834 }
|