001package org.jsoup.parser; 002 003import org.jsoup.helper.Validate; 004import org.jsoup.internal.LineMap; 005import org.jsoup.internal.SoftPool; 006import org.jsoup.internal.StringUtil; 007import org.jspecify.annotations.Nullable; 008 009import java.io.IOException; 010import java.io.UncheckedIOException; 011import java.io.Reader; 012import java.io.StringReader; 013import java.util.Arrays; 014 015/** 016 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes. 017 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an 018 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p> 019 */ 020public final class CharacterReader implements AutoCloseable { 021 static final char EOF = (char) -1; 022 private static final int MaxStringCacheLen = 12; 023 private static final int StringCacheSize = 512; 024 private String[] stringCache; // holds reused strings in this doc, to lessen garbage 025 private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations 026 027 static final int BufferSize = 1024 * 2; // visible for testing 028 static final int RefillPoint = BufferSize / 2; // when bufPos characters read, refill; visible for testing 029 private static final int RewindLimit = 1024; // the maximum we can rewind. No HTML entities can be larger than this. 030 031 private Reader reader; // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader 032 private char[] charBuf; // character buffer we consume from; filled from Reader 033 private int bufPos; // position in charBuf that's been consumed to 034 private int bufLength; // the num of characters actually buffered in charBuf, <= charBuf.length 035 private int fillPoint = 0; // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp 036 private int consumed; // how many characters total have been consumed from this CharacterReader (less the current bufPos) 037 private int bufMark = -1; // if not -1, the marked rewind position 038 private boolean readFully; // if the underlying stream has been completely read, no value in further buffering 039 040 private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer 041 042 @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions 043 044 public CharacterReader(Reader input, int sz) { 045 this(input); // sz is no longer used 046 } 047 048 public CharacterReader(Reader input) { 049 Validate.notNull(input); 050 reader = input; 051 charBuf = BufferPool.borrow(); 052 stringCache = StringPool.borrow(); 053 bufferUp(); 054 } 055 056 public CharacterReader(String input) { 057 this(new StringReader(input)); 058 } 059 060 @Override 061 public void close() { 062 if (reader == null) 063 return; 064 try { 065 reader.close(); 066 } catch (IOException ignored) { 067 } finally { 068 reader = null; 069 Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer 070 BufferPool.release(charBuf); 071 charBuf = null; 072 StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents 073 stringCache = null; 074 lineMap = null; 075 } 076 } 077 078 private void bufferUp() { 079 if (readFully || bufPos < fillPoint || bufMark != -1) 080 return; 081 doBufferUp(); // structured so bufferUp may become an intrinsic candidate 082 } 083 084 /** 085 Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException. 086 @throws UncheckedIOException if the underlying reader throws an IOException 087 */ 088 private void doBufferUp() { 089 /* 090 The flow: 091 - if read fully, or if bufPos < fillPoint, or if marked - do not fill. 092 - update readerPos (total amount consumed from this CharacterReader) += bufPos 093 - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount 094 - loop read the Reader until we fill charBuf. bufLength += read. 095 - readFully = true when read = -1 096 */ 097 consumed += bufPos; 098 bufLength -= bufPos; 099 if (bufLength > 0) 100 System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength); 101 bufPos = 0; 102 while (bufLength < BufferSize) { 103 try { 104 int read = reader.read(charBuf, bufLength, charBuf.length - bufLength); 105 if (read == -1) { 106 readFully = true; 107 break; 108 } 109 if (read == 0) { 110 break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed 111 } 112 bufLength += read; 113 } catch (IOException e) { 114 throw new UncheckedIOException(e); 115 } 116 } 117 fillPoint = Math.min(bufLength, RefillPoint); 118 119 scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking 120 lastIcSeq = null; // cache for last containsIgnoreCase(seq) 121 } 122 123 void mark() { 124 // make sure there is enough look ahead capacity 125 if (bufLength - bufPos < RewindLimit) 126 fillPoint = 0; 127 128 bufferUp(); 129 bufMark = bufPos; 130 } 131 132 void unmark() { 133 bufMark = -1; 134 } 135 136 void rewindToMark() { 137 if (bufMark == -1) 138 throw new UncheckedIOException(new IOException("Mark invalid")); 139 140 bufPos = bufMark; 141 unmark(); 142 } 143 144 /** 145 * Gets the position currently read to in the content. Starts at 0. 146 * @return current position 147 */ 148 public int pos() { 149 return consumed + bufPos; 150 } 151 152 /** Tests if the buffer has been fully read. */ 153 boolean readFully() { 154 return readFully; 155 } 156 157 /** 158 Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the 159 legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of 160 use. 161 162 @param track set tracking on|off 163 @since 1.14.3 164 */ 165 public void trackNewlines(boolean track) { 166 if (track && lineMap == null) { 167 lineMap = new LineMap(); 168 scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp 169 } 170 else if (!track) 171 lineMap = null; 172 } 173 174 /** 175 Check if the tracking of newlines is enabled. 176 @return the current newline tracking state 177 @since 1.14.3 178 */ 179 public boolean isTrackNewlines() { 180 return lineMap != null; 181 } 182 183 /** 184 Get the line map enabled by {@link #trackNewlines(boolean)}. 185 */ 186 LineMap lineMap() { 187 assert lineMap != null; 188 return lineMap; 189 } 190 191 /** 192 Get the current line number (that the reader has consumed to). Starts at line #1. 193 @return the current line number, or 1 if line tracking is not enabled. 194 @since 1.14.3 195 @see #trackNewlines(boolean) 196 */ 197 public int lineNumber() { 198 return lineNumber(pos()); 199 } 200 201 int lineNumber(int pos) { 202 if (!isTrackNewlines()) 203 return 1; 204 205 return lineMap().lineNumber(pos); 206 } 207 208 /** 209 Get the current column number (that the reader has consumed to). Starts at column #1. 210 @return the current column number 211 @since 1.14.3 212 @see #trackNewlines(boolean) 213 */ 214 public int columnNumber() { 215 return columnNumber(pos()); 216 } 217 218 int columnNumber(int pos) { 219 if (!isTrackNewlines()) 220 return pos + 1; 221 222 return lineMap().columnNumber(pos); 223 } 224 225 /** 226 Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line 227 number 5 and column number 10. 228 @return line:col position 229 @since 1.14.3 230 @see #trackNewlines(boolean) 231 */ 232 String posLineCol() { 233 return lineNumber() + ":" + columnNumber(); 234 } 235 236 /** 237 Scans the buffer for newline positions and records line starts. 238 */ 239 private void scanBufferForNewlines() { 240 if (!isTrackNewlines()) 241 return; 242 243 for (int i = bufPos; i < bufLength; i++) { 244 if (charBuf[i] == '\n') { 245 int lineStart = 1 + consumed + i; 246 lineMap().addLineStart(lineStart); 247 } 248 } 249 } 250 251 /** 252 * Tests if all the content has been read. 253 * @return true if nothing left to read. 254 */ 255 public boolean isEmpty() { 256 bufferUp(); 257 return bufPos >= bufLength; 258 } 259 260 private boolean isEmptyNoBufferUp() { 261 return bufPos >= bufLength; 262 } 263 264 /** 265 * Get the char at the current position. 266 * @return char 267 */ 268 public char current() { 269 bufferUp(); 270 return isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 271 } 272 273 /** 274 Consume one character off the queue. 275 @return first character on queue, or EOF if the queue is empty. 276 */ 277 public char consume() { 278 bufferUp(); 279 char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 280 bufPos++; 281 return val; 282 } 283 284 /** 285 Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp. 286 */ 287 void unconsume() { 288 if (bufPos < 1) 289 throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it. 290 291 bufPos--; 292 } 293 294 /** 295 * Moves the current position by one. 296 */ 297 public void advance() { 298 bufPos++; 299 } 300 301 /** 302 * Returns the number of characters between the current position and the next instance of the input char 303 * @param c scan target 304 * @return offset between current position and next instance of target. -1 if not found. 305 */ 306 int nextIndexOf(char c) { 307 // doesn't handle scanning for surrogates 308 bufferUp(); 309 for (int i = bufPos; i < bufLength; i++) { 310 if (c == charBuf[i]) 311 return i - bufPos; 312 } 313 return -1; 314 } 315 316 /** 317 * Returns the number of characters between the current position and the next instance of the input sequence 318 * 319 * @param seq scan target 320 * @return offset between current position and next instance of target. -1 if not found. 321 */ 322 int nextIndexOf(CharSequence seq) { 323 bufferUp(); 324 // doesn't handle scanning for surrogates 325 char startChar = seq.charAt(0); 326 for (int offset = bufPos; offset < bufLength; offset++) { 327 // scan to first instance of startchar: 328 if (startChar != charBuf[offset]) 329 while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ } 330 int i = offset + 1; 331 int last = i + seq.length()-1; 332 if (offset < bufLength && last <= bufLength) { 333 for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ } 334 if (i == last) // found full sequence 335 return offset - bufPos; 336 } 337 } 338 return -1; 339 } 340 341 /** 342 * Reads characters up to the specific char. 343 * @param c the delimiter 344 * @return the chars read 345 */ 346 public String consumeTo(char c) { 347 int offset = nextIndexOf(c); 348 if (offset != -1) { 349 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 350 bufPos += offset; 351 return consumed; 352 } else { 353 return consumeToEnd(); 354 } 355 } 356 357 /** 358 Reads the characters up to (but not including) the specified case-sensitive string. 359 <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the 360 length of the sequence, such that this call may be repeated. 361 @param seq the delimiter 362 @return the chars read 363 */ 364 public String consumeTo(String seq) { 365 int offset = nextIndexOf(seq); 366 if (offset != -1) { 367 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 368 bufPos += offset; 369 return consumed; 370 } else if (bufLength - bufPos < seq.length()) { 371 // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF 372 return consumeToEnd(); 373 } else { 374 // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters 375 // unread in case they contain the beginning of the search string 376 int endPos = bufLength - seq.length() + 1; 377 String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos); 378 bufPos = endPos; 379 return consumed; 380 } 381 } 382 383 /** 384 Read characters while the input predicate returns true. 385 @return characters read 386 */ 387 String consumeMatching(CharPredicate func) { 388 return consumeMatching(func, -1); 389 } 390 391 /** 392 Read characters while the input predicate returns true, up to a maximum length. 393 @param func predicate to test 394 @param maxLength maximum length to read. -1 indicates no maximum 395 @return characters read 396 */ 397 String consumeMatching(CharPredicate func, int maxLength) { 398 bufferUp(); 399 int pos = bufPos; 400 final int start = pos; 401 final int remaining = bufLength; 402 final char[] val = charBuf; 403 404 while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) { 405 pos++; 406 } 407 408 bufPos = pos; 409 return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : ""; 410 } 411 412 /** 413 Read characters until the first of any delimiters is found. 414 @param chars delimiters to scan for 415 @return characters read up to the matched delimiter. 416 */ 417 public String consumeToAny(final char... chars) { 418 bufferUp(); 419 int pos = bufPos; 420 final int start = pos; 421 final int remaining = bufLength; 422 final char[] val = charBuf; 423 424 scan: 425 while (pos < remaining) { 426 char c = val[pos]; 427 for (char seek : chars) 428 if (c == seek) break scan; 429 pos++; 430 } 431 432 return consumeRange(start, pos); 433 } 434 435 /** 436 Read characters until either delimiter is found. 437 */ 438 String consumeToAny(char c1, char c2) { 439 // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func) 440 bufferUp(); 441 int pos = bufPos; 442 final int start = pos; 443 final int remaining = bufLength; 444 final char[] val = charBuf; 445 446 while (pos < remaining) { 447 char c = val[pos]; 448 if (c == c1 || c == c2) break; 449 pos++; 450 } 451 452 return consumeRange(start, pos); 453 } 454 455 /** 456 Read characters until any delimiter is found. 457 */ 458 String consumeToAny(char c1, char c2, char c3) { 459 bufferUp(); 460 int pos = bufPos; 461 final int start = pos; 462 final int remaining = bufLength; 463 final char[] val = charBuf; 464 465 while (pos < remaining) { 466 char c = val[pos]; 467 if (c == c1 || c == c2 || c == c3) break; 468 pos++; 469 } 470 471 return consumeRange(start, pos); 472 } 473 474 String consumeToAnySorted(final char... chars) { 475 bufferUp(); 476 int pos = bufPos; 477 final int start = pos; 478 final int remaining = bufLength; 479 final char[] val = charBuf; 480 481 while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) { 482 pos++; 483 } 484 485 return consumeRange(start, pos); 486 } 487 488 String consumeData() { 489 // consumes until &, <, null 490 return consumeToAny('&', '<', TokeniserState.nullChar); 491 } 492 493 String consumeAttributeQuoted(final boolean single) { 494 // null, " or ', & 495 char quote = single ? '\'' : '"'; 496 return consumeToAny(TokeniserState.nullChar, '&', quote); 497 } 498 499 String consumeRawData() { 500 // <, null 501 return consumeToAny('<', TokeniserState.nullChar); 502 } 503 504 String consumeTagName() { 505 // '\t', '\n', '\r', '\f', ' ', '/', '>' 506 // NOTE: out of spec; does not stop and append on nullChar but eats 507 bufferUp(); 508 int pos = bufPos; 509 final int start = pos; 510 final int remaining = bufLength; 511 final char[] val = charBuf; 512 513 while (pos < remaining) { 514 char c = val[pos]; 515 switch (c) { 516 case '\t': 517 case '\n': 518 case '\r': 519 case '\f': 520 case ' ': 521 case '/': 522 case '>': 523 return consumeRange(start, pos); 524 } 525 pos++; 526 } 527 528 return consumeRange(start, pos); 529 } 530 531 String consumeToEnd() { 532 bufferUp(); 533 String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos); 534 bufPos = bufLength; 535 return data; 536 } 537 538 String consumeLetterSequence() { 539 return consumeMatching(Character::isLetter); 540 } 541 542 String consumeLetterThenDigitSequence() { 543 bufferUp(); 544 int start = bufPos; 545 while (bufPos < bufLength) { 546 if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++; 547 else break; 548 } 549 while (!isEmptyNoBufferUp()) { 550 if (StringUtil.isDigit(charBuf[bufPos])) bufPos++; 551 else break; 552 } 553 554 return cacheString(charBuf, stringCache, start, bufPos - start); 555 } 556 557 String consumeHexSequence() { 558 return consumeMatching(StringUtil::isHexDigit); 559 } 560 561 String consumeDigitSequence() { 562 return consumeMatching(c -> c >= '0' && c <= '9'); 563 } 564 565 /** 566 Complete a scan by moving the reader and returning the matched range. 567 */ 568 private String consumeRange(int start, int pos) { 569 bufPos = pos; 570 return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : ""; 571 } 572 573 boolean matches(char c) { 574 return !isEmpty() && charBuf[bufPos] == c; 575 } 576 577 boolean matches(String seq) { 578 bufferUp(); 579 int scanLength = seq.length(); 580 if (scanLength > bufLength - bufPos) 581 return false; 582 583 for (int offset = 0; offset < scanLength; offset++) 584 if (seq.charAt(offset) != charBuf[bufPos +offset]) 585 return false; 586 return true; 587 } 588 589 /** 590 Checks if the current buffer position matches the sequence case-insensitively. 591 */ 592 boolean matchesIgnoreCase(String seq) { 593 bufferUp(); 594 int scanLength = seq.length(); 595 if (scanLength > bufLength - bufPos) 596 return false; 597 598 return rangeMatchesIgnoreCase(seq, bufPos); 599 } 600 601 private boolean rangeMatchesIgnoreCase(String seq, int start) { 602 for (int offset = 0; offset < seq.length(); offset++) { 603 char scan = seq.charAt(offset); 604 char target = charBuf[start + offset]; 605 if (scan == target) continue; 606 607 scan = Character.toUpperCase(scan); 608 target = Character.toUpperCase(target); 609 if (scan != target) return false; 610 } 611 return true; 612 } 613 614 /** 615 Tests if the next character in the queue matches any of the characters in the sequence, case sensitively. 616 @param seq list of characters to check for 617 @return true if any matched, false if none did 618 */ 619 boolean matchesAny(char... seq) { 620 if (isEmpty()) 621 return false; 622 623 bufferUp(); 624 char c = charBuf[bufPos]; 625 for (char seek : seq) { 626 if (seek == c) 627 return true; 628 } 629 return false; 630 } 631 632 boolean matchesAnySorted(char[] seq) { 633 bufferUp(); 634 return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0; 635 } 636 637 /** 638 Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha 639 @return if it matches or not 640 */ 641 boolean matchesAsciiAlpha() { 642 if (isEmpty()) return false; 643 return StringUtil.isAsciiLetter(charBuf[bufPos]); 644 } 645 646 boolean matchesDigit() { 647 if (isEmpty()) return false; 648 return StringUtil.isDigit(charBuf[bufPos]); 649 } 650 651 boolean matchConsume(String seq) { 652 bufferUp(); 653 if (matches(seq)) { 654 bufPos += seq.length(); 655 return true; 656 } else { 657 return false; 658 } 659 } 660 661 boolean matchConsumeIgnoreCase(String seq) { 662 if (matchesIgnoreCase(seq)) { 663 bufPos += seq.length(); 664 return true; 665 } else { 666 return false; 667 } 668 } 669 670 // we maintain a cache of the previously scanned sequence, and return that if applicable on repeated scans. 671 // that improves the situation where there is a sequence of <p<p<p<p<p<p<p...</title> and we're bashing on the <p 672 // looking for the </title>. Resets in bufferUp() 673 @Nullable private String lastIcSeq; // scan cache 674 private int lastIcIndex; // nearest found indexOf 675 676 /** Used to check presence of </title>, </style> when we're in RCData and see a <xxx. */ 677 boolean containsIgnoreCase(String seq) { 678 bufferUp(); 679 if (seq.equals(lastIcSeq)) { 680 if (lastIcIndex == -1) return false; 681 if (lastIcIndex >= bufPos) return true; 682 } 683 lastIcSeq = seq; 684 685 int scanLength = seq.length(); 686 int maxStart = bufLength - scanLength; 687 for (int scan = bufPos; scan <= maxStart; scan++) { 688 if (rangeMatchesIgnoreCase(seq, scan)) { 689 lastIcIndex = scan; 690 return true; 691 } 692 } 693 694 lastIcIndex = -1; 695 return false; 696 } 697 698 @Override 699 public String toString() { 700 if (bufLength - bufPos < 0) return ""; 701 return new String(charBuf, bufPos, bufLength - bufPos); 702 } 703 704 /** 705 * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks. 706 * <p /> 707 * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list. 708 * That saves both having to create objects as hash keys, and running through the entry list, at the expense of 709 * some more duplicates. 710 */ 711 private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) { 712 if (count > MaxStringCacheLen) // don't cache strings that are too big 713 return new String(charBuf, start, count); 714 if (count < 1) 715 return ""; 716 717 // calculate hash: 718 int hash = 0; 719 int end = count + start; 720 for (int i = start; i < end; i++) { 721 hash = 31 * hash + charBuf[i]; 722 } 723 724 // get from cache 725 final int index = hash & StringCacheSize - 1; 726 String cached = stringCache[index]; 727 728 if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit 729 return cached; 730 else { 731 cached = new String(charBuf, start, count); 732 stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next 733 } 734 735 return cached; 736 } 737 738 /** 739 * Check if the value of the provided range equals the string. 740 */ 741 static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) { 742 if (count == cached.length()) { 743 int i = start; 744 int j = 0; 745 while (count-- != 0) { 746 if (charBuf[i++] != cached.charAt(j++)) 747 return false; 748 } 749 return true; 750 } 751 return false; 752 } 753 754 // just used for testing 755 boolean rangeEquals(final int start, final int count, final String cached) { 756 return rangeEquals(charBuf, start, count, cached); 757 } 758 759 @FunctionalInterface 760 interface CharPredicate { 761 boolean test(char c); 762 } 763}