001package org.jsoup.parser; 002 003import org.jsoup.helper.Validate; 004import org.jsoup.internal.LineMap; 005import org.jsoup.internal.SoftPool; 006import org.jsoup.internal.StringUtil; 007import org.jspecify.annotations.Nullable; 008 009import java.io.IOException; 010import java.io.UncheckedIOException; 011import java.io.Reader; 012import java.io.StringReader; 013import java.util.Arrays; 014 015import static org.jsoup.internal.Normalizer.asciiLowerCase; 016 017/** 018 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes. 019 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an 020 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p> 021 */ 022public final class CharacterReader implements AutoCloseable { 023 static final char EOF = (char) -1; 024 private static final int MaxStringCacheLen = 12; 025 private static final int StringCacheSize = 512; 026 private String[] stringCache; // holds reused strings in this doc, to lessen garbage 027 private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations 028 029 static final int BufferSize = 1024 * 2; // visible for testing 030 static final int RefillPoint = BufferSize / 2; // when bufPos characters read, refill; visible for testing 031 private static final int RewindLimit = 1024; // the maximum we can rewind. No HTML entities can be larger than this. 032 033 private Reader reader; // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader 034 private char[] charBuf; // character buffer we consume from; filled from Reader 035 private int bufPos; // position in charBuf that's been consumed to 036 private int bufLength; // the num of characters actually buffered in charBuf, <= charBuf.length 037 private int fillPoint = 0; // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp 038 private int consumed; // how many characters total have been consumed from this CharacterReader (less the current bufPos) 039 private int bufMark = -1; // if not -1, the marked rewind position 040 private boolean readFully; // if the underlying stream has been completely read, no value in further buffering 041 042 private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer 043 044 @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions 045 046 public CharacterReader(Reader input, int sz) { 047 this(input); // sz is no longer used 048 } 049 050 public CharacterReader(Reader input) { 051 Validate.notNull(input); 052 reader = input; 053 charBuf = BufferPool.borrow(); 054 stringCache = StringPool.borrow(); 055 bufferUp(); 056 } 057 058 public CharacterReader(String input) { 059 this(new StringReader(input)); 060 } 061 062 @Override 063 public void close() { 064 if (reader == null) 065 return; 066 try { 067 reader.close(); 068 } catch (IOException ignored) { 069 } finally { 070 reader = null; 071 Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer 072 BufferPool.release(charBuf); 073 charBuf = null; 074 StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents 075 stringCache = null; 076 lineMap = null; 077 } 078 } 079 080 private void bufferUp() { 081 if (readFully || bufPos < fillPoint || bufMark != -1) 082 return; 083 doBufferUp(); // structured so bufferUp may become an intrinsic candidate 084 } 085 086 /** 087 Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException. 088 @throws UncheckedIOException if the underlying reader throws an IOException 089 */ 090 private void doBufferUp() { 091 /* 092 The flow: 093 - if read fully, or if bufPos < fillPoint, or if marked - do not fill. 094 - update readerPos (total amount consumed from this CharacterReader) += bufPos 095 - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount 096 - loop read the Reader until we fill charBuf. bufLength += read. 097 - readFully = true when read = -1 098 */ 099 consumed += bufPos; 100 bufLength -= bufPos; 101 if (bufLength > 0) 102 System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength); 103 bufPos = 0; 104 while (bufLength < BufferSize) { 105 try { 106 int read = reader.read(charBuf, bufLength, charBuf.length - bufLength); 107 if (read == -1) { 108 readFully = true; 109 break; 110 } 111 if (read == 0) { 112 break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed 113 } 114 bufLength += read; 115 } catch (IOException e) { 116 throw new UncheckedIOException(e); 117 } 118 } 119 fillPoint = Math.min(bufLength, RefillPoint); 120 121 scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking 122 } 123 124 void mark() { 125 // make sure there is enough look ahead capacity 126 if (bufLength - bufPos < RewindLimit) 127 fillPoint = 0; 128 129 bufferUp(); 130 bufMark = bufPos; 131 } 132 133 void unmark() { 134 bufMark = -1; 135 } 136 137 void rewindToMark() { 138 if (bufMark == -1) 139 throw new UncheckedIOException(new IOException("Mark invalid")); 140 141 bufPos = bufMark; 142 unmark(); 143 } 144 145 /** 146 * Gets the position currently read to in the content. Starts at 0. 147 * @return current position 148 */ 149 public int pos() { 150 // consuming EOF advances to a virtual position so it can be unconsumed; don't expose that beyond the input 151 return consumed + Math.min(bufPos, bufLength); 152 } 153 154 /** 155 Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the 156 legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of 157 use. 158 159 @param track set tracking on|off 160 @since 1.14.3 161 */ 162 public void trackNewlines(boolean track) { 163 if (track && lineMap == null) { 164 lineMap = new LineMap(); 165 scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp 166 } 167 else if (!track) 168 lineMap = null; 169 } 170 171 /** 172 Check if the tracking of newlines is enabled. 173 @return the current newline tracking state 174 @since 1.14.3 175 */ 176 public boolean isTrackNewlines() { 177 return lineMap != null; 178 } 179 180 /** 181 Get the line map enabled by {@link #trackNewlines(boolean)}. 182 */ 183 LineMap lineMap() { 184 assert lineMap != null; 185 return lineMap; 186 } 187 188 /** 189 Get the current line number (that the reader has consumed to). Starts at line #1. 190 @return the current line number, or 1 if line tracking is not enabled. 191 @since 1.14.3 192 @see #trackNewlines(boolean) 193 */ 194 public int lineNumber() { 195 return lineNumber(pos()); 196 } 197 198 int lineNumber(int pos) { 199 if (!isTrackNewlines()) 200 return 1; 201 202 return lineMap().lineNumber(pos); 203 } 204 205 /** 206 Get the current column number (that the reader has consumed to). Starts at column #1. 207 @return the current column number 208 @since 1.14.3 209 @see #trackNewlines(boolean) 210 */ 211 public int columnNumber() { 212 return columnNumber(pos()); 213 } 214 215 int columnNumber(int pos) { 216 if (!isTrackNewlines()) 217 return pos + 1; 218 219 return lineMap().columnNumber(pos); 220 } 221 222 /** 223 Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line 224 number 5 and column number 10. 225 @return line:col position 226 @since 1.14.3 227 @see #trackNewlines(boolean) 228 */ 229 String posLineCol() { 230 return lineNumber() + ":" + columnNumber(); 231 } 232 233 /** 234 Scans the buffer for newline positions and records line starts. 235 */ 236 private void scanBufferForNewlines() { 237 if (!isTrackNewlines()) 238 return; 239 240 for (int i = bufPos; i < bufLength; i++) { 241 if (charBuf[i] == '\n') { 242 int lineStart = 1 + consumed + i; 243 lineMap().addLineStart(lineStart); 244 } 245 } 246 } 247 248 /** 249 * Tests if all the content has been read. 250 * @return true if nothing left to read. 251 */ 252 public boolean isEmpty() { 253 bufferUp(); 254 return bufPos >= bufLength; 255 } 256 257 private boolean isEmptyNoBufferUp() { 258 return bufPos >= bufLength; 259 } 260 261 /** 262 * Get the char at the current position. 263 * @return char 264 */ 265 public char current() { 266 bufferUp(); 267 return isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 268 } 269 270 /** 271 Consume one character off the queue. 272 @return first character on queue, or EOF if the queue is empty. 273 */ 274 public char consume() { 275 bufferUp(); 276 char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 277 bufPos++; 278 return val; 279 } 280 281 /** 282 Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp. 283 */ 284 void unconsume() { 285 if (bufPos < 1) 286 throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it. 287 288 bufPos--; 289 } 290 291 /** 292 * Moves the current position by one. 293 */ 294 public void advance() { 295 bufPos++; 296 } 297 298 /** 299 * Returns the number of characters between the current position and the next instance of the input char 300 * @param c scan target 301 * @return offset between current position and next instance of target. -1 if not found. 302 */ 303 int nextIndexOf(char c) { 304 // doesn't handle scanning for surrogates 305 bufferUp(); 306 for (int i = bufPos; i < bufLength; i++) { 307 if (c == charBuf[i]) 308 return i - bufPos; 309 } 310 return -1; 311 } 312 313 /** 314 * Returns the number of characters between the current position and the next instance of the input sequence 315 * 316 * @param seq scan target 317 * @return offset between current position and next instance of target. -1 if not found. 318 */ 319 int nextIndexOf(CharSequence seq) { 320 bufferUp(); 321 // doesn't handle scanning for surrogates 322 char startChar = seq.charAt(0); 323 for (int offset = bufPos; offset < bufLength; offset++) { 324 // scan to first instance of startchar: 325 if (startChar != charBuf[offset]) 326 while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ } 327 int i = offset + 1; 328 int last = i + seq.length()-1; 329 if (offset < bufLength && last <= bufLength) { 330 for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ } 331 if (i == last) // found full sequence 332 return offset - bufPos; 333 } 334 } 335 return -1; 336 } 337 338 /** 339 * Reads characters up to the specific char. 340 * @param c the delimiter 341 * @return the chars read 342 */ 343 public String consumeTo(char c) { 344 int offset = nextIndexOf(c); 345 if (offset != -1) { 346 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 347 bufPos += offset; 348 return consumed; 349 } else { 350 return consumeToEnd(); 351 } 352 } 353 354 /** 355 Reads the characters up to (but not including) the specified case-sensitive string. 356 <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the 357 length of the sequence, such that this call may be repeated. 358 @param seq the delimiter 359 @return the chars read 360 */ 361 public String consumeTo(String seq) { 362 int offset = nextIndexOf(seq); 363 if (offset != -1) { 364 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 365 bufPos += offset; 366 return consumed; 367 } else if (bufLength - bufPos < seq.length()) { 368 // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF 369 return consumeToEnd(); 370 } else { 371 // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters 372 // unread in case they contain the beginning of the search string 373 int endPos = bufLength - seq.length() + 1; 374 String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos); 375 bufPos = endPos; 376 return consumed; 377 } 378 } 379 380 /** 381 Read characters while the input predicate returns true. 382 @return characters read 383 */ 384 String consumeMatching(CharPredicate func) { 385 return consumeMatching(func, -1); 386 } 387 388 /** 389 Read characters while the input predicate returns true, up to a maximum length. 390 @param func predicate to test 391 @param maxLength maximum length to read. -1 indicates no maximum 392 @return characters read 393 */ 394 String consumeMatching(CharPredicate func, int maxLength) { 395 bufferUp(); 396 int pos = bufPos; 397 final int start = pos; 398 final int remaining = bufLength; 399 final char[] val = charBuf; 400 401 while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) { 402 pos++; 403 } 404 405 bufPos = pos; 406 return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : ""; 407 } 408 409 /** 410 Read characters until the first of any delimiters is found. 411 @param chars delimiters to scan for 412 @return characters read up to the matched delimiter. 413 */ 414 public String consumeToAny(final char... chars) { 415 bufferUp(); 416 int pos = bufPos; 417 final int start = pos; 418 final int remaining = bufLength; 419 final char[] val = charBuf; 420 421 scan: 422 while (pos < remaining) { 423 char c = val[pos]; 424 for (char seek : chars) 425 if (c == seek) break scan; 426 pos++; 427 } 428 429 return consumeRange(start, pos); 430 } 431 432 /** 433 Read characters until either delimiter is found. 434 */ 435 String consumeToAny(char c1, char c2) { 436 // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func) 437 bufferUp(); 438 int pos = bufPos; 439 final int start = pos; 440 final int remaining = bufLength; 441 final char[] val = charBuf; 442 443 while (pos < remaining) { 444 char c = val[pos]; 445 if (c == c1 || c == c2) break; 446 pos++; 447 } 448 449 return consumeRange(start, pos); 450 } 451 452 /** 453 Read characters until any delimiter is found. 454 */ 455 String consumeToAny(char c1, char c2, char c3) { 456 bufferUp(); 457 int pos = bufPos; 458 final int start = pos; 459 final int remaining = bufLength; 460 final char[] val = charBuf; 461 462 while (pos < remaining) { 463 char c = val[pos]; 464 if (c == c1 || c == c2 || c == c3) break; 465 pos++; 466 } 467 468 return consumeRange(start, pos); 469 } 470 471 String consumeToAnySorted(final char... chars) { 472 bufferUp(); 473 int pos = bufPos; 474 final int start = pos; 475 final int remaining = bufLength; 476 final char[] val = charBuf; 477 478 while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) { 479 pos++; 480 } 481 482 return consumeRange(start, pos); 483 } 484 485 /** Read HTML whitespace. */ 486 String consumeWhitespace() { 487 bufferUp(); 488 int start = bufPos; 489 int pos = start; 490 while (pos < bufLength && StringUtil.isWhitespace(charBuf[pos])) pos++; 491 return consumeRange(start, pos); 492 } 493 494 String consumeData() { 495 // consumes until &, <, null 496 return consumeToAny('&', '<', TokeniserState.nullChar); 497 } 498 499 String consumeAttributeQuoted(final boolean single) { 500 // null, " or ', & 501 char quote = single ? '\'' : '"'; 502 return consumeToAny(TokeniserState.nullChar, '&', quote); 503 } 504 505 String consumeRawData() { 506 // <, null 507 return consumeToAny('<', TokeniserState.nullChar); 508 } 509 510 String consumeTagName() { 511 // '\t', '\n', '\r', '\f', ' ', '/', '>' 512 // NOTE: out of spec; does not stop and append on nullChar but eats 513 bufferUp(); 514 int pos = bufPos; 515 final int start = pos; 516 final int remaining = bufLength; 517 final char[] val = charBuf; 518 519 while (pos < remaining) { 520 char c = val[pos]; 521 switch (c) { 522 case '\t': 523 case '\n': 524 case '\r': 525 case '\f': 526 case ' ': 527 case '/': 528 case '>': 529 return consumeRange(start, pos); 530 } 531 pos++; 532 } 533 534 return consumeRange(start, pos); 535 } 536 537 String consumeToEnd() { 538 bufferUp(); 539 String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos); 540 bufPos = bufLength; 541 return data; 542 } 543 544 /** Consumes ASCII letters used by the HTML tokenizer's name states. */ 545 String consumeLetterSequence() { 546 return consumeMatching(StringUtil::isAsciiLetter); 547 } 548 549 String consumeLetterThenDigitSequence() { 550 bufferUp(); 551 int start = bufPos; 552 while (bufPos < bufLength) { 553 if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++; 554 else break; 555 } 556 while (!isEmptyNoBufferUp()) { 557 if (StringUtil.isDigit(charBuf[bufPos])) bufPos++; 558 else break; 559 } 560 561 return cacheString(charBuf, stringCache, start, bufPos - start); 562 } 563 564 String consumeHexSequence() { 565 return consumeMatching(StringUtil::isHexDigit); 566 } 567 568 String consumeDigitSequence() { 569 return consumeMatching(c -> c >= '0' && c <= '9'); 570 } 571 572 /** 573 Complete a scan by moving the reader and returning the matched range. 574 */ 575 private String consumeRange(int start, int pos) { 576 bufPos = pos; 577 return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : ""; 578 } 579 580 boolean matches(char c) { 581 return !isEmpty() && charBuf[bufPos] == c; 582 } 583 584 boolean matches(String seq) { 585 bufferUp(); 586 int scanLength = seq.length(); 587 if (scanLength > bufLength - bufPos) 588 return false; 589 590 for (int offset = 0; offset < scanLength; offset++) 591 if (seq.charAt(offset) != charBuf[bufPos +offset]) 592 return false; 593 return true; 594 } 595 596 /** 597 Checks if the current buffer position matches the sequence using ASCII case-insensitive matching. 598 */ 599 boolean matchesIgnoreCase(String seq) { 600 bufferUp(); 601 int scanLength = seq.length(); 602 if (scanLength > bufLength - bufPos) 603 return false; 604 605 return rangeMatchesIgnoreCase(seq, bufPos); 606 } 607 608 private boolean rangeMatchesIgnoreCase(String seq, int start) { 609 for (int offset = 0; offset < seq.length(); offset++) { 610 char scan = seq.charAt(offset); 611 char target = charBuf[start + offset]; 612 if (scan == target) continue; 613 614 scan = asciiLowerCase(scan); 615 target = asciiLowerCase(target); 616 if (scan != target) return false; 617 } 618 return true; 619 } 620 621 /** 622 Tests if the next character in the queue matches any of the characters in the sequence, case sensitively. 623 @param seq list of characters to check for 624 @return true if any matched, false if none did 625 */ 626 boolean matchesAny(char... seq) { 627 if (isEmpty()) 628 return false; 629 630 bufferUp(); 631 char c = charBuf[bufPos]; 632 for (char seek : seq) { 633 if (seek == c) 634 return true; 635 } 636 return false; 637 } 638 639 boolean matchesAnySorted(char[] seq) { 640 bufferUp(); 641 return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0; 642 } 643 644 /** 645 Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha 646 @return if it matches or not 647 */ 648 boolean matchesAsciiAlpha() { 649 if (isEmpty()) return false; 650 return StringUtil.isAsciiLetter(charBuf[bufPos]); 651 } 652 653 boolean matchesDigit() { 654 if (isEmpty()) return false; 655 return StringUtil.isDigit(charBuf[bufPos]); 656 } 657 658 boolean matchConsume(String seq) { 659 bufferUp(); 660 if (matches(seq)) { 661 bufPos += seq.length(); 662 return true; 663 } else { 664 return false; 665 } 666 } 667 668 boolean matchConsumeIgnoreCase(String seq) { 669 if (matchesIgnoreCase(seq)) { 670 bufPos += seq.length(); 671 return true; 672 } else { 673 return false; 674 } 675 } 676 677 @Override 678 public String toString() { 679 if (bufLength - bufPos < 0) return ""; 680 return new String(charBuf, bufPos, bufLength - bufPos); 681 } 682 683 /** 684 * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks. 685 * <p /> 686 * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list. 687 * That saves both having to create objects as hash keys, and running through the entry list, at the expense of 688 * some more duplicates. 689 */ 690 private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) { 691 if (count > MaxStringCacheLen) // don't cache strings that are too big 692 return new String(charBuf, start, count); 693 if (count < 1) 694 return ""; 695 696 // calculate hash: 697 int hash = 0; 698 int end = count + start; 699 for (int i = start; i < end; i++) { 700 hash = 31 * hash + charBuf[i]; 701 } 702 703 // get from cache 704 final int index = hash & StringCacheSize - 1; 705 String cached = stringCache[index]; 706 707 if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit 708 return cached; 709 else { 710 cached = new String(charBuf, start, count); 711 stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next 712 } 713 714 return cached; 715 } 716 717 /** 718 * Check if the value of the provided range equals the string. 719 */ 720 static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) { 721 if (count == cached.length()) { 722 int i = start; 723 int j = 0; 724 while (count-- != 0) { 725 if (charBuf[i++] != cached.charAt(j++)) 726 return false; 727 } 728 return true; 729 } 730 return false; 731 } 732 733 // just used for testing 734 boolean rangeEquals(final int start, final int count, final String cached) { 735 return rangeEquals(charBuf, start, count, cached); 736 } 737 738 @FunctionalInterface 739 interface CharPredicate { 740 boolean test(char c); 741 } 742}