6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package javafx.css;
27
28 import com.sun.javafx.css.parser.Token;
29
30 import java.io.CharArrayReader;
31 import java.io.Reader;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collections;
35 import java.util.List;
36 import static org.junit.Assert.*;
37 import static org.junit.Assert.assertEquals;
38
39 import org.junit.Test;
40
41
42 public class CssLexerTest {
43
44 public CssLexerTest() {
45 }
46
47 private void checkTokens(List<Token> resultTokens, Token... expectedTokens)
48 throws org.junit.ComparisonFailure {
49
50 if (expectedTokens.length != resultTokens.size()) {
51 throw new org.junit.ComparisonFailure(
52 "lengths do not match",
53 Arrays.toString(expectedTokens),
54 resultTokens.toString()
55 );
56 }
57
58 for (int n = 0; n<expectedTokens.length; n++) {
59
60 final Token result = resultTokens.get(n);
61 final Token expected = expectedTokens[n];
62
63 if (expected.getType() != result.getType()) {
64 throw new org.junit.ComparisonFailure(
65 "token " + n + " types do not match",
66 Arrays.toString(expectedTokens),
67 resultTokens.toString()
68 );
69 }
70
71 final String expectedText = expected.getText();
72 final String resultText = result.getText();
73
74 if (expectedText == null ? resultText != null : !expectedText.equals(resultText)) {
75 throw new org.junit.ComparisonFailure(
76 "token " + n + " text does not match",
77 Arrays.toString(expectedTokens),
78 resultTokens.toString()
79 );
80 }
81 }
82 }
83
84 List<Token> getTokens(String string) {
85
86 Reader reader = new CharArrayReader(string.toCharArray());
87 final CssLexer lexer = new CssLexer();
88 lexer.setReader(reader);
89
90 final List<Token> tokens = new ArrayList<Token>();
91
92 Token token = null;
93 do {
94 token = lexer.nextToken();
95 tokens.add(token);
96 } while (token.getType() != Token.EOF);
97
98 return Collections.unmodifiableList(tokens);
99 }
100
101 private void lexDigitsWithUnits(String units, int type) throws org.junit.ComparisonFailure {
102
103 checkTokens(getTokens("123"+units), new Token(type, "123"+units), Token.EOF_TOKEN);
104 checkTokens(getTokens("123.45"+units), new Token(type, "123.45"+units), Token.EOF_TOKEN);
105 checkTokens(getTokens(".45"+units), new Token(type, ".45"+units), Token.EOF_TOKEN);
106 checkTokens(getTokens("-123"+units), new Token(type, "-123"+units), Token.EOF_TOKEN);
107 checkTokens(getTokens("-.45"+units), new Token(type, "-.45"+units), Token.EOF_TOKEN);
108 checkTokens(getTokens("+123"+units), new Token(type, "+123"+units), Token.EOF_TOKEN);
109 checkTokens(getTokens("+.45"+units), new Token(type, "+.45"+units), Token.EOF_TOKEN);
110 }
111
112 @Test
113 public void testLexValidDigits() {
114 lexDigitsWithUnits("", CssLexer.NUMBER);
115 }
116
117 @Test
118 public void testLexValidDigitsWithCM() {
119 lexDigitsWithUnits("cm", CssLexer.CM);
120 // case should be ignored
121 lexDigitsWithUnits("cM", CssLexer.CM);
122 }
123 @Test
124 public void testLexValidDigitsWithDEG() {
125 lexDigitsWithUnits("deg", CssLexer.DEG);
126 // case should be ignored
127 lexDigitsWithUnits("dEg", CssLexer.DEG);
128 }
129 @Test
130 public void testLexValidDigitsWithEM() {
131 lexDigitsWithUnits("em", CssLexer.EMS);
132 // case should be ignored
133 lexDigitsWithUnits("Em", CssLexer.EMS);
134 }
135 @Test
136 public void testLexValidDigitsWithEX() {
137 lexDigitsWithUnits("ex", CssLexer.EXS);
138 // case should be ignored
139 lexDigitsWithUnits("Ex", CssLexer.EXS);
140 }
141 @Test
142 public void testLexValidDigitsWithGRAD() {
143 lexDigitsWithUnits("grad", CssLexer.GRAD);
144 // case should be ignored
145 lexDigitsWithUnits("gRad", CssLexer.GRAD);
146 }
147 @Test
148 public void testLexValidDigitsWithIN() {
149 lexDigitsWithUnits("in", CssLexer.IN);
150 // case should be ignored
151 lexDigitsWithUnits("In", CssLexer.IN);
152 }
153 @Test
154 public void testLexValidDigitsWithMM() {
155 lexDigitsWithUnits("mm", CssLexer.MM);
156 // case should be ignored
157 lexDigitsWithUnits("mM", CssLexer.MM);
158 }
159 @Test
160 public void testLexValidDigitsWithPC() {
161 lexDigitsWithUnits("pc", CssLexer.PC);
162 // case should be ignored
163 lexDigitsWithUnits("Pc", CssLexer.PC);
164 }
165 @Test
166 public void testLexValidDigitsWithPT() {
167 lexDigitsWithUnits("pt", CssLexer.PT);
168 // case should be ignored
169 lexDigitsWithUnits("PT", CssLexer.PT);
170 }
171 @Test
172 public void testLexValidDigitsWithPX() {
173 lexDigitsWithUnits("px", CssLexer.PX);
174 // case should be ignored
175 lexDigitsWithUnits("Px", CssLexer.PX);
176 }
177 @Test
178 public void testLexValidDigitsWithRAD() {
179 lexDigitsWithUnits("rad", CssLexer.RAD);
180 // case should be ignored
181 lexDigitsWithUnits("RaD", CssLexer.RAD);
182 }
183 @Test
184 public void testLexValidDigitsWithTURN() {
185 lexDigitsWithUnits("turn", CssLexer.TURN);
186 // case should be ignored
187 lexDigitsWithUnits("TurN", CssLexer.TURN);
188 }
189 @Test
190 public void testLexValidDigitsWithS() {
191 lexDigitsWithUnits("s", CssLexer.SECONDS);
192 // case should be ignored
193 lexDigitsWithUnits("S", CssLexer.SECONDS);
194 }
195 @Test
196 public void testLexValidDigitsWithMS() {
197 lexDigitsWithUnits("ms", CssLexer.MS);
198 // case should be ignored
199 lexDigitsWithUnits("mS", CssLexer.MS);
200 }
201 @Test
202 public void testLexValidDigitsWithPCT() {
203 lexDigitsWithUnits("%", CssLexer.PERCENTAGE);
204 }
205 @Test
206 public void testLexValidDigitsWithBadUnits() {
207 lexDigitsWithUnits("xyzzy", Token.INVALID);
208 }
209 @Test
210 public void textLexValidDigitsValidDigits() {
211 checkTokens(
212 getTokens("foo: 10pt; bar: 20%;"),
213 new Token(CssLexer.IDENT, "foo"),
214 new Token(CssLexer.COLON, ":"),
215 new Token(CssLexer.WS, " "),
216 new Token(CssLexer.PT, "10pt"),
217 new Token(CssLexer.SEMI, ";"),
218 new Token(CssLexer.WS, " "),
219 new Token(CssLexer.IDENT, "bar"),
220 new Token(CssLexer.COLON, ":"),
221 new Token(CssLexer.WS, " "),
222 new Token(CssLexer.PERCENTAGE, "20%"),
223 new Token(CssLexer.SEMI, ";"),
224 Token.EOF_TOKEN
225 );
226 }
227 @Test
228 public void textLexInvalidDigitsValidDigits() {
229 checkTokens(
230 getTokens("foo: 10pz; bar: 20%;"),
231 new Token(CssLexer.IDENT, "foo"),
232 new Token(CssLexer.COLON, ":"),
233 new Token(CssLexer.WS, " "),
234 new Token(Token.INVALID, "10pz"),
235 new Token(CssLexer.SEMI, ";"),
236 new Token(CssLexer.WS, " "),
237 new Token(CssLexer.IDENT, "bar"),
238 new Token(CssLexer.COLON, ":"),
239 new Token(CssLexer.WS, " "),
240 new Token(CssLexer.PERCENTAGE, "20%"),
241 new Token(CssLexer.SEMI, ";"),
242 Token.EOF_TOKEN
243 );
244 }
245 @Test
246 public void textLexValidDigitsBangImportant() {
247 checkTokens(
248 getTokens("foo: 10pt !important;"),
249 new Token(CssLexer.IDENT, "foo"),
250 new Token(CssLexer.COLON, ":"),
251 new Token(CssLexer.WS, " "),
252 new Token(CssLexer.PT, "10pt"),
253 new Token(CssLexer.WS, " "),
254 new Token(CssLexer.IMPORTANT_SYM, "!important"),
255 new Token(CssLexer.SEMI, ";"),
256 Token.EOF_TOKEN
257 );
258 }
259 @Test
260 public void textLexInvalidDigitsBangImportant() {
261 checkTokens(
262 getTokens("foo: 10pz !important;"),
263 new Token(CssLexer.IDENT, "foo"),
264 new Token(CssLexer.COLON, ":"),
265 new Token(CssLexer.WS, " "),
266 new Token(Token.INVALID, "10pz"),
267 new Token(CssLexer.WS, " "),
268 new Token(CssLexer.IMPORTANT_SYM, "!important"),
269 new Token(CssLexer.SEMI, ";"),
270 Token.EOF_TOKEN
271 );
272 }
273 @Test
274 public void textLexValidDigitsInSequence() {
275 checkTokens(
276 getTokens("-1 0px 1pt .5em;"),
277 new Token(CssLexer.NUMBER, "-1"),
278 new Token(CssLexer.WS, " "),
279 new Token(CssLexer.PX, "0px"),
280 new Token(CssLexer.WS, " "),
281 new Token(CssLexer.PT, "1pt"),
282 new Token(CssLexer.WS, " "),
283 new Token(CssLexer.EMS, ".5em"),
284 new Token(CssLexer.SEMI, ";"),
285 Token.EOF_TOKEN
286 );
287 }
288 @Test
289 public void textLexInvalidDigitsInSequence() {
290 checkTokens(
291 getTokens("-1 0px 1pz .5em;"),
292 new Token(CssLexer.NUMBER, "-1"),
293 new Token(CssLexer.WS, " "),
294 new Token(CssLexer.PX, "0px"),
295 new Token(CssLexer.WS, " "),
296 new Token(Token.INVALID, "1pz"),
297 new Token(CssLexer.WS, " "),
298 new Token(CssLexer.EMS, ".5em"),
299 new Token(CssLexer.SEMI, ";"),
300 Token.EOF_TOKEN
301 );
302 }
303
304 @Test
305 public void testTokenOffset() {
306
307 String str = "a: b;";
308 // [?][0] = line
309 // [?][1] = offset
310 Token[] expected = {
311 new Token(CssLexer.IDENT, "a", 1, 0),
312 new Token(CssLexer.COLON, ":", 1, 1),
313 new Token(CssLexer.WS, " ", 1, 2),
314 new Token(CssLexer.IDENT, "b", 1, 3),
315 new Token(CssLexer.SEMI, ";", 1, 4),
316 Token.EOF_TOKEN
317 };
318
319 List<Token> tlist = getTokens(str);
320 checkTokens(tlist, expected);
321
322 for(int n=0; n<tlist.size(); n++) {
323 Token tok = tlist.get(n);
324 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
325 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
326 }
327
328 }
329
330 @Test
331 public void testTokenLineAndOffsetWithCR() {
332
333 String str = "a: b;\rc: d;";
334 // [?][0] = line
335 // [?][1] = offset
336 Token[] expected = {
337 new Token(CssLexer.IDENT, "a", 1, 0),
338 new Token(CssLexer.COLON, ":", 1, 1),
339 new Token(CssLexer.WS, " ", 1, 2),
340 new Token(CssLexer.IDENT, "b", 1, 3),
341 new Token(CssLexer.SEMI, ";", 1, 4),
342 new Token(CssLexer.NL, "\\r", 1, 5),
343 new Token(CssLexer.IDENT, "c", 2, 0),
344 new Token(CssLexer.COLON, ":", 2, 1),
345 new Token(CssLexer.WS, " ", 2, 2),
346 new Token(CssLexer.IDENT, "d", 2, 3),
347 new Token(CssLexer.SEMI, ";", 2, 4),
348 Token.EOF_TOKEN
349 };
350
351 List<Token> tlist = getTokens(str);
352 checkTokens(tlist, expected);
353
354 for(int n=0; n<tlist.size(); n++) {
355 Token tok = tlist.get(n);
356 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
357 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
358 }
359
360 }
361
362 @Test
363 public void testTokenLineAndOffsetWithLF() {
364
365 String str = "a: b;\nc: d;";
366 // [?][0] = line
367 // [?][1] = offset
368 Token[] expected = {
369 new Token(CssLexer.IDENT, "a", 1, 0),
370 new Token(CssLexer.COLON, ":", 1, 1),
371 new Token(CssLexer.WS, " ", 1, 2),
372 new Token(CssLexer.IDENT, "b", 1, 3),
373 new Token(CssLexer.SEMI, ";", 1, 4),
374 new Token(CssLexer.NL, "\\n", 1, 5),
375 new Token(CssLexer.IDENT, "c", 2, 0),
376 new Token(CssLexer.COLON, ":", 2, 1),
377 new Token(CssLexer.WS, " ", 2, 2),
378 new Token(CssLexer.IDENT, "d", 2, 3),
379 new Token(CssLexer.SEMI, ";", 2, 4),
380 Token.EOF_TOKEN
381 };
382
383 List<Token> tlist = getTokens(str);
384 checkTokens(tlist, expected);
385
386 for(int n=0; n<tlist.size(); n++) {
387 Token tok = tlist.get(n);
388 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
389 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
390 }
391
392 }
393
394 @Test
395 public void testTokenLineAndOffsetWithCRLF() {
396 // 012345 01234
397 String str = "a: b;\r\nc: d;";
398 // [?][0] = line
399 // [?][1] = offset
400 Token[] expected = {
401 new Token(CssLexer.IDENT, "a", 1, 0),
402 new Token(CssLexer.COLON, ":", 1, 1),
403 new Token(CssLexer.WS, " ", 1, 2),
404 new Token(CssLexer.IDENT, "b", 1, 3),
405 new Token(CssLexer.SEMI, ";", 1, 4),
406 new Token(CssLexer.NL, "\\r\\n", 1, 5),
407 new Token(CssLexer.IDENT, "c", 2, 0),
408 new Token(CssLexer.COLON, ":", 2, 1),
409 new Token(CssLexer.WS, " ", 2, 2),
410 new Token(CssLexer.IDENT, "d", 2, 3),
411 new Token(CssLexer.SEMI, ";", 2, 4),
412 Token.EOF_TOKEN
413 };
414
415 List<Token> tlist = getTokens(str);
416 checkTokens(tlist, expected);
417
418 for(int n=0; n<tlist.size(); n++) {
419 Token tok = tlist.get(n);
420 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
421 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
422 }
423
424 }
425
426 @Test
427 public void testTokenOffsetWithEmbeddedComment() {
428 // 0123456789012345
429 String str = "a: /*comment*/b;";
430 // [?][0] = line
431 // [?][1] = offset
432 Token[] expected = {
433 new Token(CssLexer.IDENT, "a", 1, 0),
434 new Token(CssLexer.COLON, ":", 1, 1),
435 new Token(CssLexer.WS, " ", 1, 2),
436 new Token(CssLexer.IDENT, "b", 1, 14),
437 new Token(CssLexer.SEMI, ";", 1, 15),
438 Token.EOF_TOKEN
439 };
440
441 List<Token> tlist = getTokens(str);
442 checkTokens(tlist, expected);
443
444 for(int n=0; n<tlist.size(); n++) {
445 Token tok = tlist.get(n);
446 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
447 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
448 }
449 }
450
451 @Test
452 public void testTokenLineAndOffsetWithLeadingComment() {
453 // 012345678901 01234
454 String str = "/*comment*/\na: b;";
455 // [?][0] = line
456 // [?][1] = offset
457 Token[] expected = {
458 new Token(CssLexer.NL, "\\n", 1, 11),
459 new Token(CssLexer.IDENT, "a", 2, 0),
460 new Token(CssLexer.COLON, ":", 2, 1),
461 new Token(CssLexer.WS, " ", 2, 2),
462 new Token(CssLexer.IDENT, "b", 2, 3),
463 new Token(CssLexer.SEMI, ";", 2, 4),
464 Token.EOF_TOKEN
465 };
466
467 List<Token> tlist = getTokens(str);
468 checkTokens(tlist, expected);
469
470 for(int n=0; n<tlist.size(); n++) {
471 Token tok = tlist.get(n);
472 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
473 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
474 }
475 }
476
477 @Test
478 public void testTokenOffsetWithFunction() {
479 // 01234567890
480 String str = "a: b(arg);";
481 // [?][0] = line
482 // [?][1] = offset
483 Token[] expected = {
484 new Token(CssLexer.IDENT, "a", 1, 0),
485 new Token(CssLexer.COLON, ":", 1, 1),
486 new Token(CssLexer.WS, " ", 1, 2),
487 new Token(CssLexer.IDENT, "b", 1, 3),
488 new Token(CssLexer.LPAREN, "(", 1, 4),
489 new Token(CssLexer.IDENT, "arg", 1, 5),
490 new Token(CssLexer.RPAREN, ")", 1, 8),
491 new Token(CssLexer.SEMI, ";", 1, 9),
492 Token.EOF_TOKEN
493 };
494
495 List<Token> tlist = getTokens(str);
496 checkTokens(tlist, expected);
497
498 for(int n=0; n<tlist.size(); n++) {
499 Token tok = tlist.get(n);
500 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
501 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
502 }
503 }
504
505 @Test
506 public void testTokenOffsetWithHash() {
507 // 01234567890
508 String str = "a: #012345;";
509 // [?][0] = line
510 // [?][1] = offset
511 Token[] expected = {
512 new Token(CssLexer.IDENT, "a", 1, 0),
513 new Token(CssLexer.COLON, ":", 1, 1),
514 new Token(CssLexer.WS, " ", 1, 2),
515 new Token(CssLexer.HASH, "#012345", 1, 3),
516 new Token(CssLexer.SEMI, ";", 1, 10),
517 Token.EOF_TOKEN
518 };
519
520 List<Token> tlist = getTokens(str);
521 checkTokens(tlist, expected);
522
523 for(int n=0; n<tlist.size(); n++) {
524 Token tok = tlist.get(n);
525 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
526 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
527 }
528 }
529
530 @Test
531 public void testTokenOffsetWithDigits() {
532 // 01234567890
533 String str = "a: 123.45;";
534 // [?][0] = line
535 // [?][1] = offset
536 Token[] expected = {
537 new Token(CssLexer.IDENT, "a", 1, 0),
538 new Token(CssLexer.COLON, ":", 1, 1),
539 new Token(CssLexer.WS, " ", 1, 2),
540 new Token(CssLexer.NUMBER, "123.45", 1, 3),
541 new Token(CssLexer.SEMI, ";", 1, 9),
542 Token.EOF_TOKEN
543 };
544
545 List<Token> tlist = getTokens(str);
546 checkTokens(tlist, expected);
547
548 for(int n=0; n<tlist.size(); n++) {
549 Token tok = tlist.get(n);
550 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
551 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
552 }
553 }
554
555 @Test
556 public void testTokenOffsetWithBangImportant() {
557 // 0123456789012345
558 String str = "a: b !important;";
559 // [?][0] = line
560 // [?][1] = offset
561 Token[] expected = {
562 new Token(CssLexer.IDENT, "a", 1, 0),
563 new Token(CssLexer.COLON, ":", 1, 1),
564 new Token(CssLexer.WS, " ", 1, 2),
565 new Token(CssLexer.IDENT, "b", 1, 3),
566 new Token(CssLexer.WS, " ", 1, 4),
567 new Token(CssLexer.IMPORTANT_SYM, "!important", 1, 5),
568 new Token(CssLexer.SEMI, ";", 1, 15),
569 Token.EOF_TOKEN
570 };
571
572 List<Token> tlist = getTokens(str);
573 checkTokens(tlist, expected);
574
575 for(int n=0; n<tlist.size(); n++) {
576 Token tok = tlist.get(n);
577 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
578 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
579 }
580 }
581
582 @Test
583 public void testTokenOffsetWithSkip() {
584 // 0123456789012345
585 String str = "a: b !imporzant;";
586 // [?][0] = line
587 // [?][1] = offset
588 Token[] expected = {
589 new Token(CssLexer.IDENT, "a", 1, 0),
590 new Token(CssLexer.COLON, ":", 1, 1),
591 new Token(CssLexer.WS, " ", 1, 2),
592 new Token(CssLexer.IDENT, "b", 1, 3),
593 new Token(CssLexer.WS, " ", 1, 4),
594 new Token(Token.SKIP, "!imporz", 1, 5),
595 new Token(CssLexer.SEMI, ";", 1, 15),
596 Token.EOF_TOKEN
597 };
598
599 List<Token> tlist = getTokens(str);
600 checkTokens(tlist, expected);
601
602 for(int n=0; n<tlist.size(); n++) {
603 Token tok = tlist.get(n);
604 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
605 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
606 }
607 }
608
609 @Test
610 public void testTokenOffsetWithInvalid() {
611 // 0123456789012345
612 String str = "a: 1pz;";
613 // [?][0] = line
614 // [?][1] = offset
615 Token[] expected = {
616 new Token(CssLexer.IDENT, "a", 1, 0),
617 new Token(CssLexer.COLON, ":", 1, 1),
618 new Token(CssLexer.WS, " ", 1, 2),
619 new Token(Token.INVALID, "1pz", 1, 3),
620 new Token(CssLexer.SEMI, ";", 1, 6),
621 Token.EOF_TOKEN
622 };
623
624 List<Token> tlist = getTokens(str);
625 checkTokens(tlist, expected);
626
627 for(int n=0; n<tlist.size(); n++) {
628 Token tok = tlist.get(n);
629 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
630 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
631 }
632 }
633
634 @Test
635 public void testTokenLineAndOffsetMoreFully() {
636 // 1 2 3 4
637 // 012345678901 0123456789012345 012345678 0
638 String str = "/*comment*/\n*.foo#bar:baz {\n\ta: 1em;\n}";
639 // [?][0] = line
640 // [?][1] = offset
641 Token[] expected = {
642 new Token(CssLexer.NL, "\\n", 1, 11),
643 new Token(CssLexer.STAR, "*", 2, 0),
644 new Token(CssLexer.DOT, ".", 2, 1),
645 new Token(CssLexer.IDENT, "foo", 2, 2),
646 new Token(CssLexer.HASH, "#bar", 2, 5),
647 new Token(CssLexer.COLON, ":", 2, 9),
648 new Token(CssLexer.IDENT, "baz", 2, 10),
649 new Token(CssLexer.WS, " ", 2, 13),
650 new Token(CssLexer.LBRACE, "{", 2, 14),
651 new Token(CssLexer.NL, "\\n", 2, 15),
652 new Token(CssLexer.WS, "\t", 3, 0),
653 new Token(CssLexer.IDENT, "a", 3, 1),
654 new Token(CssLexer.COLON, ":", 3, 2),
655 new Token(CssLexer.WS, " ", 3, 3),
656 new Token(CssLexer.EMS, "1em", 3, 4),
657 new Token(CssLexer.SEMI, ";", 3, 7),
658 new Token(CssLexer.NL, "\\n", 3, 8),
659 new Token(CssLexer.RBRACE, "}", 4, 0),
660 Token.EOF_TOKEN
661 };
662
663 List<Token> tlist = getTokens(str);
664 checkTokens(tlist, expected);
665
666 for(int n=0; n<tlist.size(); n++) {
667 Token tok = tlist.get(n);
668 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
669 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
670 }
671 }
672
673 @Test
674 public void testScanUrl() {
675
676 // 1 2 3 4
677 // 01234567890101234567890123450123456780123456789
678 String str = "url(http://foo.bar.com/fonts/serif/fubar.ttf)";
679 Token[] expected = new Token[]{
680 new Token(CssLexer.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
681 Token.EOF_TOKEN
682 };
683
684 List<Token> tlist = getTokens(str);
685 checkTokens(tlist, expected);
686
687 for(int n=0; n<tlist.size(); n++) {
688 Token tok = tlist.get(n);
689 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
690 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
691 }
692
693 }
694
695 @Test
696 public void testScanUrlWithWhiteSpace() {
697
698 // 1 2 3 4
699 // 01234567890101234567890123450123456780123456789
700 String str = "url( http://foo.bar.com/fonts/serif/fubar.ttf\t)";
701 Token[] expected = new Token[]{
702 new Token(CssLexer.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
703 Token.EOF_TOKEN
704 };
705
706 List<Token> tlist = getTokens(str);
707 checkTokens(tlist, expected);
708
709 for(int n=0; n<tlist.size(); n++) {
710 Token tok = tlist.get(n);
711 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
712 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
713 }
714
715 }
716
717 @Test
718 public void testScanQuotedUrlWithWhiteSpace() {
719
720 // 1 2 3 4
721 // 01234567890101234567890123450123456780123456789
722 String str = "url( 'http://foo.bar.com/fonts/serif/fubar.ttf'\t)";
723 Token[] expected = new Token[]{
724 new Token(CssLexer.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
725 Token.EOF_TOKEN
726 };
727
728 List<Token> tlist = getTokens(str);
729 checkTokens(tlist, expected);
730
731 for(int n=0; n<tlist.size(); n++) {
732 Token tok = tlist.get(n);
733 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
734 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
735 }
736
737 }
738
739 @Test
740 public void testScanQuotedUrl() {
741
742 // 1 2 3 4
743 // 01234567890101234567890123450123456780123456789
744 String str = "url(\"http://foo.bar.com/fonts/serif/fubar.ttf\")";
745 Token[] expected = new Token[]{
746 new Token(CssLexer.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
747 Token.EOF_TOKEN
748 };
749
750 List<Token> tlist = getTokens(str);
751 checkTokens(tlist, expected);
752
753 for(int n=0; n<tlist.size(); n++) {
754 Token tok = tlist.get(n);
755 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
756 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
757 }
758
759 }
760
761 @Test
762 public void testScanUrlWithEscapes() {
763
764 // 1 2 3 4
765 // 01234567890101234567890123450123456780123456789
766 String str = "url(http://foo.bar.com/fonts/true\\ type/fubar.ttf)";
767 Token[] expected = new Token[]{
768 new Token(CssLexer.URL, "http://foo.bar.com/fonts/true type/fubar.ttf", 1, 0),
769 Token.EOF_TOKEN
770 };
771
772 List<Token> tlist = getTokens(str);
773 checkTokens(tlist, expected);
774
775 for(int n=0; n<tlist.size(); n++) {
776 Token tok = tlist.get(n);
777 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
778 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
779 }
780
781 }
782
783 @Test
784 public void testScanQuotedUrlWithEscapes() {
785
786 // 1 2 3 4
787 // 01234567890101234567890123450123456780123456789
788 String str = "url(\"http://foo.bar.com/fonts/true\\ type/fubar.ttf\")";
789 Token[] expected = new Token[]{
790 new Token(CssLexer.URL, "http://foo.bar.com/fonts/true type/fubar.ttf", 1, 0),
791 Token.EOF_TOKEN
792 };
793
794 List<Token> tlist = getTokens(str);
795 checkTokens(tlist, expected);
796
797 for(int n=0; n<tlist.size(); n++) {
798 Token tok = tlist.get(n);
799 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
800 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
801 }
802
803 }
804
805 @Test
806 public void testScanUrlWithSyntaxError() {
807
808 // 1 2 3 4
809 // 01234567890101234567890123450123456780123456789
810 String str = "url(http://foo.bar.com/fonts/true'type/fubar.ttf)";
811 Token[] expected = new Token[]{
812 new Token(Token.INVALID, "http://foo.bar.com/fonts/true", 1, 0),
813 Token.EOF_TOKEN
814 };
815
816 List<Token> tlist = getTokens(str);
817 checkTokens(tlist, expected);
818
819 for(int n=0; n<tlist.size(); n++) {
820 Token tok = tlist.get(n);
821 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
822 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
823 }
824
825 }
826
827 @Test
828 public void testScanQuotedUrlWithSyntaxError() {
829
830 // 1 2 3 4
831 // 01234567890101234567890123450123456780123456789
832 String str = "url('http://foo.bar.com/fonts/true\rtype/fubar.ttf')";
833 Token[] expected = new Token[]{
834 new Token(Token.INVALID, "http://foo.bar.com/fonts/true", 2, 0),
835 Token.EOF_TOKEN
836 };
837
838 List<Token> tlist = getTokens(str);
839 checkTokens(tlist, expected);
840
841 for(int n=0; n<tlist.size(); n++) {
842 Token tok = tlist.get(n);
843 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
844 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
845 }
846
847 }
848
849 }
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package test.javafx.css;
27
28 import com.sun.javafx.css.parser.Token;
29 import com.sun.javafx.css.parser.TokenShim;
30
31 import java.io.CharArrayReader;
32 import java.io.Reader;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collections;
36 import java.util.List;
37 import javafx.css.CssLexerShim;
38 import static org.junit.Assert.*;
39 import static org.junit.Assert.assertEquals;
40
41 import org.junit.Test;
42
43
44 public class CssLexerTest {
45
46 public CssLexerTest() {
47 }
48
49 private void checkTokens(List<TokenShim> resultTokens, TokenShim... expectedTokens)
50 throws org.junit.ComparisonFailure {
51
52 if (expectedTokens.length != resultTokens.size()) {
53 throw new org.junit.ComparisonFailure(
54 "lengths do not match",
55 Arrays.toString(expectedTokens),
56 resultTokens.toString()
57 );
58 }
59
60 for (int n = 0; n<expectedTokens.length; n++) {
61
62 final TokenShim result = resultTokens.get(n);
63 final TokenShim expected = expectedTokens[n];
64
65 if (expected.getType() != result.getType()) {
66 throw new org.junit.ComparisonFailure(
67 "token " + n + " types do not match",
68 Arrays.toString(expectedTokens),
69 resultTokens.toString()
70 );
71 }
72
73 final String expectedText = expected.getText();
74 final String resultText = result.getText();
75
76 if (expectedText == null ? resultText != null : !expectedText.equals(resultText)) {
77 throw new org.junit.ComparisonFailure(
78 "token " + n + " text does not match",
79 Arrays.toString(expectedTokens),
80 resultTokens.toString()
81 );
82 }
83 }
84 }
85
86 List<TokenShim> getTokens(String string) {
87
88 Reader reader = new CharArrayReader(string.toCharArray());
89 final CssLexerShim lexer = new CssLexerShim();
90 lexer.setReader(reader);
91
92 final List<TokenShim> tokens = new ArrayList<TokenShim>();
93
94 TokenShim token = null;
95 do {
96 token = lexer.nextToken();
97 tokens.add(token);
98 } while (token.getType() != Token.EOF);
99
100 return Collections.unmodifiableList(tokens);
101 }
102
103 private void lexDigitsWithUnits(String units, int type) throws org.junit.ComparisonFailure {
104
105 checkTokens(getTokens("123"+units), new TokenShim(type, "123"+units), TokenShim.EOF_TOKEN);
106 checkTokens(getTokens("123.45"+units), new TokenShim(type, "123.45"+units), TokenShim.EOF_TOKEN);
107 checkTokens(getTokens(".45"+units), new TokenShim(type, ".45"+units), TokenShim.EOF_TOKEN);
108 checkTokens(getTokens("-123"+units), new TokenShim(type, "-123"+units), TokenShim.EOF_TOKEN);
109 checkTokens(getTokens("-.45"+units), new TokenShim(type, "-.45"+units), TokenShim.EOF_TOKEN);
110 checkTokens(getTokens("+123"+units), new TokenShim(type, "+123"+units), TokenShim.EOF_TOKEN);
111 checkTokens(getTokens("+.45"+units), new TokenShim(type, "+.45"+units), TokenShim.EOF_TOKEN);
112 }
113
114 @Test
115 public void testLexValidDigits() {
116 lexDigitsWithUnits("", CssLexerShim.NUMBER);
117 }
118
119 @Test
120 public void testLexValidDigitsWithCM() {
121 lexDigitsWithUnits("cm", CssLexerShim.CM);
122 // case should be ignored
123 lexDigitsWithUnits("cM", CssLexerShim.CM);
124 }
125 @Test
126 public void testLexValidDigitsWithDEG() {
127 lexDigitsWithUnits("deg", CssLexerShim.DEG);
128 // case should be ignored
129 lexDigitsWithUnits("dEg", CssLexerShim.DEG);
130 }
131 @Test
132 public void testLexValidDigitsWithEM() {
133 lexDigitsWithUnits("em", CssLexerShim.EMS);
134 // case should be ignored
135 lexDigitsWithUnits("Em", CssLexerShim.EMS);
136 }
137 @Test
138 public void testLexValidDigitsWithEX() {
139 lexDigitsWithUnits("ex", CssLexerShim.EXS);
140 // case should be ignored
141 lexDigitsWithUnits("Ex", CssLexerShim.EXS);
142 }
143 @Test
144 public void testLexValidDigitsWithGRAD() {
145 lexDigitsWithUnits("grad", CssLexerShim.GRAD);
146 // case should be ignored
147 lexDigitsWithUnits("gRad", CssLexerShim.GRAD);
148 }
149 @Test
150 public void testLexValidDigitsWithIN() {
151 lexDigitsWithUnits("in", CssLexerShim.IN);
152 // case should be ignored
153 lexDigitsWithUnits("In", CssLexerShim.IN);
154 }
155 @Test
156 public void testLexValidDigitsWithMM() {
157 lexDigitsWithUnits("mm", CssLexerShim.MM);
158 // case should be ignored
159 lexDigitsWithUnits("mM", CssLexerShim.MM);
160 }
161 @Test
162 public void testLexValidDigitsWithPC() {
163 lexDigitsWithUnits("pc", CssLexerShim.PC);
164 // case should be ignored
165 lexDigitsWithUnits("Pc", CssLexerShim.PC);
166 }
167 @Test
168 public void testLexValidDigitsWithPT() {
169 lexDigitsWithUnits("pt", CssLexerShim.PT);
170 // case should be ignored
171 lexDigitsWithUnits("PT", CssLexerShim.PT);
172 }
173 @Test
174 public void testLexValidDigitsWithPX() {
175 lexDigitsWithUnits("px", CssLexerShim.PX);
176 // case should be ignored
177 lexDigitsWithUnits("Px", CssLexerShim.PX);
178 }
179 @Test
180 public void testLexValidDigitsWithRAD() {
181 lexDigitsWithUnits("rad", CssLexerShim.RAD);
182 // case should be ignored
183 lexDigitsWithUnits("RaD", CssLexerShim.RAD);
184 }
185 @Test
186 public void testLexValidDigitsWithTURN() {
187 lexDigitsWithUnits("turn", CssLexerShim.TURN);
188 // case should be ignored
189 lexDigitsWithUnits("TurN", CssLexerShim.TURN);
190 }
191 @Test
192 public void testLexValidDigitsWithS() {
193 lexDigitsWithUnits("s", CssLexerShim.SECONDS);
194 // case should be ignored
195 lexDigitsWithUnits("S", CssLexerShim.SECONDS);
196 }
197 @Test
198 public void testLexValidDigitsWithMS() {
199 lexDigitsWithUnits("ms", CssLexerShim.MS);
200 // case should be ignored
201 lexDigitsWithUnits("mS", CssLexerShim.MS);
202 }
203 @Test
204 public void testLexValidDigitsWithPCT() {
205 lexDigitsWithUnits("%", CssLexerShim.PERCENTAGE);
206 }
207 @Test
208 public void testLexValidDigitsWithBadUnits() {
209 lexDigitsWithUnits("xyzzy", Token.INVALID);
210 }
211 @Test
212 public void textLexValidDigitsValidDigits() {
213 checkTokens(
214 getTokens("foo: 10pt; bar: 20%;"),
215 new TokenShim(CssLexerShim.IDENT, "foo"),
216 new TokenShim(CssLexerShim.COLON, ":"),
217 new TokenShim(CssLexerShim.WS, " "),
218 new TokenShim(CssLexerShim.PT, "10pt"),
219 new TokenShim(CssLexerShim.SEMI, ";"),
220 new TokenShim(CssLexerShim.WS, " "),
221 new TokenShim(CssLexerShim.IDENT, "bar"),
222 new TokenShim(CssLexerShim.COLON, ":"),
223 new TokenShim(CssLexerShim.WS, " "),
224 new TokenShim(CssLexerShim.PERCENTAGE, "20%"),
225 new TokenShim(CssLexerShim.SEMI, ";"),
226 TokenShim.EOF_TOKEN
227 );
228 }
229 @Test
230 public void textLexInvalidDigitsValidDigits() {
231 checkTokens(
232 getTokens("foo: 10pz; bar: 20%;"),
233 new TokenShim(CssLexerShim.IDENT, "foo"),
234 new TokenShim(CssLexerShim.COLON, ":"),
235 new TokenShim(CssLexerShim.WS, " "),
236 new TokenShim(Token.INVALID, "10pz"),
237 new TokenShim(CssLexerShim.SEMI, ";"),
238 new TokenShim(CssLexerShim.WS, " "),
239 new TokenShim(CssLexerShim.IDENT, "bar"),
240 new TokenShim(CssLexerShim.COLON, ":"),
241 new TokenShim(CssLexerShim.WS, " "),
242 new TokenShim(CssLexerShim.PERCENTAGE, "20%"),
243 new TokenShim(CssLexerShim.SEMI, ";"),
244 TokenShim.EOF_TOKEN
245 );
246 }
247 @Test
248 public void textLexValidDigitsBangImportant() {
249 checkTokens(
250 getTokens("foo: 10pt !important;"),
251 new TokenShim(CssLexerShim.IDENT, "foo"),
252 new TokenShim(CssLexerShim.COLON, ":"),
253 new TokenShim(CssLexerShim.WS, " "),
254 new TokenShim(CssLexerShim.PT, "10pt"),
255 new TokenShim(CssLexerShim.WS, " "),
256 new TokenShim(CssLexerShim.IMPORTANT_SYM, "!important"),
257 new TokenShim(CssLexerShim.SEMI, ";"),
258 TokenShim.EOF_TOKEN
259 );
260 }
261 @Test
262 public void textLexInvalidDigitsBangImportant() {
263 checkTokens(
264 getTokens("foo: 10pz !important;"),
265 new TokenShim(CssLexerShim.IDENT, "foo"),
266 new TokenShim(CssLexerShim.COLON, ":"),
267 new TokenShim(CssLexerShim.WS, " "),
268 new TokenShim(Token.INVALID, "10pz"),
269 new TokenShim(CssLexerShim.WS, " "),
270 new TokenShim(CssLexerShim.IMPORTANT_SYM, "!important"),
271 new TokenShim(CssLexerShim.SEMI, ";"),
272 TokenShim.EOF_TOKEN
273 );
274 }
275 @Test
276 public void textLexValidDigitsInSequence() {
277 checkTokens(
278 getTokens("-1 0px 1pt .5em;"),
279 new TokenShim(CssLexerShim.NUMBER, "-1"),
280 new TokenShim(CssLexerShim.WS, " "),
281 new TokenShim(CssLexerShim.PX, "0px"),
282 new TokenShim(CssLexerShim.WS, " "),
283 new TokenShim(CssLexerShim.PT, "1pt"),
284 new TokenShim(CssLexerShim.WS, " "),
285 new TokenShim(CssLexerShim.EMS, ".5em"),
286 new TokenShim(CssLexerShim.SEMI, ";"),
287 TokenShim.EOF_TOKEN
288 );
289 }
290 @Test
291 public void textLexInvalidDigitsInSequence() {
292 checkTokens(
293 getTokens("-1 0px 1pz .5em;"),
294 new TokenShim(CssLexerShim.NUMBER, "-1"),
295 new TokenShim(CssLexerShim.WS, " "),
296 new TokenShim(CssLexerShim.PX, "0px"),
297 new TokenShim(CssLexerShim.WS, " "),
298 new TokenShim(Token.INVALID, "1pz"),
299 new TokenShim(CssLexerShim.WS, " "),
300 new TokenShim(CssLexerShim.EMS, ".5em"),
301 new TokenShim(CssLexerShim.SEMI, ";"),
302 TokenShim.EOF_TOKEN
303 );
304 }
305
306 @Test
307 public void testTokenOffset() {
308
309 String str = "a: b;";
310 // [?][0] = line
311 // [?][1] = offset
312 TokenShim[] expected = {
313 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
314 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
315 new TokenShim(CssLexerShim.WS, " ", 1, 2),
316 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
317 new TokenShim(CssLexerShim.SEMI, ";", 1, 4),
318 TokenShim.EOF_TOKEN
319 };
320
321 List<TokenShim> tlist = getTokens(str);
322 checkTokens(tlist, expected);
323
324 for(int n=0; n<tlist.size(); n++) {
325 TokenShim tok = tlist.get(n);
326 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
327 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
328 }
329
330 }
331
332 @Test
333 public void testTokenLineAndOffsetWithCR() {
334
335 String str = "a: b;\rc: d;";
336 // [?][0] = line
337 // [?][1] = offset
338 TokenShim[] expected = {
339 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
340 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
341 new TokenShim(CssLexerShim.WS, " ", 1, 2),
342 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
343 new TokenShim(CssLexerShim.SEMI, ";", 1, 4),
344 new TokenShim(CssLexerShim.NL, "\\r", 1, 5),
345 new TokenShim(CssLexerShim.IDENT, "c", 2, 0),
346 new TokenShim(CssLexerShim.COLON, ":", 2, 1),
347 new TokenShim(CssLexerShim.WS, " ", 2, 2),
348 new TokenShim(CssLexerShim.IDENT, "d", 2, 3),
349 new TokenShim(CssLexerShim.SEMI, ";", 2, 4),
350 TokenShim.EOF_TOKEN
351 };
352
353 List<TokenShim> tlist = getTokens(str);
354 checkTokens(tlist, expected);
355
356 for(int n=0; n<tlist.size(); n++) {
357 TokenShim tok = tlist.get(n);
358 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
359 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
360 }
361
362 }
363
364 @Test
365 public void testTokenLineAndOffsetWithLF() {
366
367 String str = "a: b;\nc: d;";
368 // [?][0] = line
369 // [?][1] = offset
370 TokenShim[] expected = {
371 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
372 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
373 new TokenShim(CssLexerShim.WS, " ", 1, 2),
374 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
375 new TokenShim(CssLexerShim.SEMI, ";", 1, 4),
376 new TokenShim(CssLexerShim.NL, "\\n", 1, 5),
377 new TokenShim(CssLexerShim.IDENT, "c", 2, 0),
378 new TokenShim(CssLexerShim.COLON, ":", 2, 1),
379 new TokenShim(CssLexerShim.WS, " ", 2, 2),
380 new TokenShim(CssLexerShim.IDENT, "d", 2, 3),
381 new TokenShim(CssLexerShim.SEMI, ";", 2, 4),
382 TokenShim.EOF_TOKEN
383 };
384
385 List<TokenShim> tlist = getTokens(str);
386 checkTokens(tlist, expected);
387
388 for(int n=0; n<tlist.size(); n++) {
389 TokenShim tok = tlist.get(n);
390 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
391 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
392 }
393
394 }
395
396 @Test
397 public void testTokenLineAndOffsetWithCRLF() {
398 // 012345 01234
399 String str = "a: b;\r\nc: d;";
400 // [?][0] = line
401 // [?][1] = offset
402 TokenShim[] expected = {
403 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
404 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
405 new TokenShim(CssLexerShim.WS, " ", 1, 2),
406 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
407 new TokenShim(CssLexerShim.SEMI, ";", 1, 4),
408 new TokenShim(CssLexerShim.NL, "\\r\\n", 1, 5),
409 new TokenShim(CssLexerShim.IDENT, "c", 2, 0),
410 new TokenShim(CssLexerShim.COLON, ":", 2, 1),
411 new TokenShim(CssLexerShim.WS, " ", 2, 2),
412 new TokenShim(CssLexerShim.IDENT, "d", 2, 3),
413 new TokenShim(CssLexerShim.SEMI, ";", 2, 4),
414 TokenShim.EOF_TOKEN
415 };
416
417 List<TokenShim> tlist = getTokens(str);
418 checkTokens(tlist, expected);
419
420 for(int n=0; n<tlist.size(); n++) {
421 TokenShim tok = tlist.get(n);
422 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
423 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
424 }
425
426 }
427
428 @Test
429 public void testTokenOffsetWithEmbeddedComment() {
430 // 0123456789012345
431 String str = "a: /*comment*/b;";
432 // [?][0] = line
433 // [?][1] = offset
434 TokenShim[] expected = {
435 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
436 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
437 new TokenShim(CssLexerShim.WS, " ", 1, 2),
438 new TokenShim(CssLexerShim.IDENT, "b", 1, 14),
439 new TokenShim(CssLexerShim.SEMI, ";", 1, 15),
440 TokenShim.EOF_TOKEN
441 };
442
443 List<TokenShim> tlist = getTokens(str);
444 checkTokens(tlist, expected);
445
446 for(int n=0; n<tlist.size(); n++) {
447 TokenShim tok = tlist.get(n);
448 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
449 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
450 }
451 }
452
453 @Test
454 public void testTokenLineAndOffsetWithLeadingComment() {
455 // 012345678901 01234
456 String str = "/*comment*/\na: b;";
457 // [?][0] = line
458 // [?][1] = offset
459 TokenShim[] expected = {
460 new TokenShim(CssLexerShim.NL, "\\n", 1, 11),
461 new TokenShim(CssLexerShim.IDENT, "a", 2, 0),
462 new TokenShim(CssLexerShim.COLON, ":", 2, 1),
463 new TokenShim(CssLexerShim.WS, " ", 2, 2),
464 new TokenShim(CssLexerShim.IDENT, "b", 2, 3),
465 new TokenShim(CssLexerShim.SEMI, ";", 2, 4),
466 TokenShim.EOF_TOKEN
467 };
468
469 List<TokenShim> tlist = getTokens(str);
470 checkTokens(tlist, expected);
471
472 for(int n=0; n<tlist.size(); n++) {
473 TokenShim tok = tlist.get(n);
474 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
475 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
476 }
477 }
478
479 @Test
480 public void testTokenOffsetWithFunction() {
481 // 01234567890
482 String str = "a: b(arg);";
483 // [?][0] = line
484 // [?][1] = offset
485 TokenShim[] expected = {
486 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
487 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
488 new TokenShim(CssLexerShim.WS, " ", 1, 2),
489 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
490 new TokenShim(CssLexerShim.LPAREN, "(", 1, 4),
491 new TokenShim(CssLexerShim.IDENT, "arg", 1, 5),
492 new TokenShim(CssLexerShim.RPAREN, ")", 1, 8),
493 new TokenShim(CssLexerShim.SEMI, ";", 1, 9),
494 TokenShim.EOF_TOKEN
495 };
496
497 List<TokenShim> tlist = getTokens(str);
498 checkTokens(tlist, expected);
499
500 for(int n=0; n<tlist.size(); n++) {
501 TokenShim tok = tlist.get(n);
502 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
503 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
504 }
505 }
506
507 @Test
508 public void testTokenOffsetWithHash() {
509 // 01234567890
510 String str = "a: #012345;";
511 // [?][0] = line
512 // [?][1] = offset
513 TokenShim[] expected = {
514 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
515 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
516 new TokenShim(CssLexerShim.WS, " ", 1, 2),
517 new TokenShim(CssLexerShim.HASH, "#012345", 1, 3),
518 new TokenShim(CssLexerShim.SEMI, ";", 1, 10),
519 TokenShim.EOF_TOKEN
520 };
521
522 List<TokenShim> tlist = getTokens(str);
523 checkTokens(tlist, expected);
524
525 for(int n=0; n<tlist.size(); n++) {
526 TokenShim tok = tlist.get(n);
527 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
528 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
529 }
530 }
531
532 @Test
533 public void testTokenOffsetWithDigits() {
534 // 01234567890
535 String str = "a: 123.45;";
536 // [?][0] = line
537 // [?][1] = offset
538 TokenShim[] expected = {
539 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
540 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
541 new TokenShim(CssLexerShim.WS, " ", 1, 2),
542 new TokenShim(CssLexerShim.NUMBER, "123.45", 1, 3),
543 new TokenShim(CssLexerShim.SEMI, ";", 1, 9),
544 TokenShim.EOF_TOKEN
545 };
546
547 List<TokenShim> tlist = getTokens(str);
548 checkTokens(tlist, expected);
549
550 for(int n=0; n<tlist.size(); n++) {
551 TokenShim tok = tlist.get(n);
552 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
553 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
554 }
555 }
556
557 @Test
558 public void testTokenOffsetWithBangImportant() {
559 // 0123456789012345
560 String str = "a: b !important;";
561 // [?][0] = line
562 // [?][1] = offset
563 TokenShim[] expected = {
564 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
565 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
566 new TokenShim(CssLexerShim.WS, " ", 1, 2),
567 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
568 new TokenShim(CssLexerShim.WS, " ", 1, 4),
569 new TokenShim(CssLexerShim.IMPORTANT_SYM, "!important", 1, 5),
570 new TokenShim(CssLexerShim.SEMI, ";", 1, 15),
571 TokenShim.EOF_TOKEN
572 };
573
574 List<TokenShim> tlist = getTokens(str);
575 checkTokens(tlist, expected);
576
577 for(int n=0; n<tlist.size(); n++) {
578 TokenShim tok = tlist.get(n);
579 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
580 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
581 }
582 }
583
584 @Test
585 public void testTokenOffsetWithSkip() {
586 // 0123456789012345
587 String str = "a: b !imporzant;";
588 // [?][0] = line
589 // [?][1] = offset
590 TokenShim[] expected = {
591 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
592 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
593 new TokenShim(CssLexerShim.WS, " ", 1, 2),
594 new TokenShim(CssLexerShim.IDENT, "b", 1, 3),
595 new TokenShim(CssLexerShim.WS, " ", 1, 4),
596 new TokenShim(Token.SKIP, "!imporz", 1, 5),
597 new TokenShim(CssLexerShim.SEMI, ";", 1, 15),
598 TokenShim.EOF_TOKEN
599 };
600
601 List<TokenShim> tlist = getTokens(str);
602 checkTokens(tlist, expected);
603
604 for(int n=0; n<tlist.size(); n++) {
605 TokenShim tok = tlist.get(n);
606 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
607 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
608 }
609 }
610
611 @Test
612 public void testTokenOffsetWithInvalid() {
613 // 0123456789012345
614 String str = "a: 1pz;";
615 // [?][0] = line
616 // [?][1] = offset
617 TokenShim[] expected = {
618 new TokenShim(CssLexerShim.IDENT, "a", 1, 0),
619 new TokenShim(CssLexerShim.COLON, ":", 1, 1),
620 new TokenShim(CssLexerShim.WS, " ", 1, 2),
621 new TokenShim(Token.INVALID, "1pz", 1, 3),
622 new TokenShim(CssLexerShim.SEMI, ";", 1, 6),
623 TokenShim.EOF_TOKEN
624 };
625
626 List<TokenShim> tlist = getTokens(str);
627 checkTokens(tlist, expected);
628
629 for(int n=0; n<tlist.size(); n++) {
630 TokenShim tok = tlist.get(n);
631 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
632 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
633 }
634 }
635
636 @Test
637 public void testTokenLineAndOffsetMoreFully() {
638 // 1 2 3 4
639 // 012345678901 0123456789012345 012345678 0
640 String str = "/*comment*/\n*.foo#bar:baz {\n\ta: 1em;\n}";
641 // [?][0] = line
642 // [?][1] = offset
643 TokenShim[] expected = {
644 new TokenShim(CssLexerShim.NL, "\\n", 1, 11),
645 new TokenShim(CssLexerShim.STAR, "*", 2, 0),
646 new TokenShim(CssLexerShim.DOT, ".", 2, 1),
647 new TokenShim(CssLexerShim.IDENT, "foo", 2, 2),
648 new TokenShim(CssLexerShim.HASH, "#bar", 2, 5),
649 new TokenShim(CssLexerShim.COLON, ":", 2, 9),
650 new TokenShim(CssLexerShim.IDENT, "baz", 2, 10),
651 new TokenShim(CssLexerShim.WS, " ", 2, 13),
652 new TokenShim(CssLexerShim.LBRACE, "{", 2, 14),
653 new TokenShim(CssLexerShim.NL, "\\n", 2, 15),
654 new TokenShim(CssLexerShim.WS, "\t", 3, 0),
655 new TokenShim(CssLexerShim.IDENT, "a", 3, 1),
656 new TokenShim(CssLexerShim.COLON, ":", 3, 2),
657 new TokenShim(CssLexerShim.WS, " ", 3, 3),
658 new TokenShim(CssLexerShim.EMS, "1em", 3, 4),
659 new TokenShim(CssLexerShim.SEMI, ";", 3, 7),
660 new TokenShim(CssLexerShim.NL, "\\n", 3, 8),
661 new TokenShim(CssLexerShim.RBRACE, "}", 4, 0),
662 TokenShim.EOF_TOKEN
663 };
664
665 List<TokenShim> tlist = getTokens(str);
666 checkTokens(tlist, expected);
667
668 for(int n=0; n<tlist.size(); n++) {
669 TokenShim tok = tlist.get(n);
670 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
671 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
672 }
673 }
674
675 @Test
676 public void testScanUrl() {
677
678 // 1 2 3 4
679 // 01234567890101234567890123450123456780123456789
680 String str = "url(http://foo.bar.com/fonts/serif/fubar.ttf)";
681 TokenShim[] expected = new TokenShim[]{
682 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
683 TokenShim.EOF_TOKEN
684 };
685
686 List<TokenShim> tlist = getTokens(str);
687 checkTokens(tlist, expected);
688
689 for(int n=0; n<tlist.size(); n++) {
690 TokenShim tok = tlist.get(n);
691 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
692 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
693 }
694
695 }
696
697 @Test
698 public void testScanUrlWithWhiteSpace() {
699
700 // 1 2 3 4
701 // 01234567890101234567890123450123456780123456789
702 String str = "url( http://foo.bar.com/fonts/serif/fubar.ttf\t)";
703 TokenShim[] expected = new TokenShim[]{
704 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
705 TokenShim.EOF_TOKEN
706 };
707
708 List<TokenShim> tlist = getTokens(str);
709 checkTokens(tlist, expected);
710
711 for(int n=0; n<tlist.size(); n++) {
712 TokenShim tok = tlist.get(n);
713 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
714 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
715 }
716
717 }
718
719 @Test
720 public void testScanQuotedUrlWithWhiteSpace() {
721
722 // 1 2 3 4
723 // 01234567890101234567890123450123456780123456789
724 String str = "url( 'http://foo.bar.com/fonts/serif/fubar.ttf'\t)";
725 TokenShim[] expected = new TokenShim[]{
726 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
727 TokenShim.EOF_TOKEN
728 };
729
730 List<TokenShim> tlist = getTokens(str);
731 checkTokens(tlist, expected);
732
733 for(int n=0; n<tlist.size(); n++) {
734 TokenShim tok = tlist.get(n);
735 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
736 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
737 }
738
739 }
740
741 @Test
742 public void testScanQuotedUrl() {
743
744 // 1 2 3 4
745 // 01234567890101234567890123450123456780123456789
746 String str = "url(\"http://foo.bar.com/fonts/serif/fubar.ttf\")";
747 TokenShim[] expected = new TokenShim[]{
748 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/serif/fubar.ttf", 1, 0),
749 TokenShim.EOF_TOKEN
750 };
751
752 List<TokenShim> tlist = getTokens(str);
753 checkTokens(tlist, expected);
754
755 for(int n=0; n<tlist.size(); n++) {
756 TokenShim tok = tlist.get(n);
757 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
758 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
759 }
760
761 }
762
763 @Test
764 public void testScanUrlWithEscapes() {
765
766 // 1 2 3 4
767 // 01234567890101234567890123450123456780123456789
768 String str = "url(http://foo.bar.com/fonts/true\\ type/fubar.ttf)";
769 TokenShim[] expected = new TokenShim[]{
770 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/true type/fubar.ttf", 1, 0),
771 TokenShim.EOF_TOKEN
772 };
773
774 List<TokenShim> tlist = getTokens(str);
775 checkTokens(tlist, expected);
776
777 for(int n=0; n<tlist.size(); n++) {
778 TokenShim tok = tlist.get(n);
779 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
780 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
781 }
782
783 }
784
785 @Test
786 public void testScanQuotedUrlWithEscapes() {
787
788 // 1 2 3 4
789 // 01234567890101234567890123450123456780123456789
790 String str = "url(\"http://foo.bar.com/fonts/true\\ type/fubar.ttf\")";
791 TokenShim[] expected = new TokenShim[]{
792 new TokenShim(CssLexerShim.URL, "http://foo.bar.com/fonts/true type/fubar.ttf", 1, 0),
793 TokenShim.EOF_TOKEN
794 };
795
796 List<TokenShim> tlist = getTokens(str);
797 checkTokens(tlist, expected);
798
799 for(int n=0; n<tlist.size(); n++) {
800 TokenShim tok = tlist.get(n);
801 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
802 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
803 }
804
805 }
806
807 @Test
808 public void testScanUrlWithSyntaxError() {
809
810 // 1 2 3 4
811 // 01234567890101234567890123450123456780123456789
812 String str = "url(http://foo.bar.com/fonts/true'type/fubar.ttf)";
813 TokenShim[] expected = new TokenShim[]{
814 new TokenShim(Token.INVALID, "http://foo.bar.com/fonts/true", 1, 0),
815 TokenShim.EOF_TOKEN
816 };
817
818 List<TokenShim> tlist = getTokens(str);
819 checkTokens(tlist, expected);
820
821 for(int n=0; n<tlist.size(); n++) {
822 TokenShim tok = tlist.get(n);
823 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
824 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
825 }
826
827 }
828
829 @Test
830 public void testScanQuotedUrlWithSyntaxError() {
831
832 // 1 2 3 4
833 // 01234567890101234567890123450123456780123456789
834 String str = "url('http://foo.bar.com/fonts/true\rtype/fubar.ttf')";
835 TokenShim[] expected = new TokenShim[]{
836 new TokenShim(Token.INVALID, "http://foo.bar.com/fonts/true", 2, 0),
837 TokenShim.EOF_TOKEN
838 };
839
840 List<TokenShim> tlist = getTokens(str);
841 checkTokens(tlist, expected);
842
843 for(int n=0; n<tlist.size(); n++) {
844 TokenShim tok = tlist.get(n);
845 assertEquals("bad line. tok="+tok, expected[n].getLine(), tok.getLine());
846 assertEquals("bad offset. tok="+tok, expected[n].getOffset(), tok.getOffset());
847 }
848
849 }
850
851 }
|