001package org.jsoup.parser; 002 003import org.jsoup.helper.Validate; 004import org.jsoup.internal.LineMap; 005import org.jsoup.internal.SoftPool; 006import org.jsoup.internal.StringUtil; 007import org.jspecify.annotations.Nullable; 008 009import java.io.IOException; 010import java.io.UncheckedIOException; 011import java.io.Reader; 012import java.io.StringReader; 013import java.util.Arrays; 014 015/** 016 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes. 017 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an 018 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p> 019 */ 020public final class CharacterReader implements AutoCloseable { 021 static final char EOF = (char) -1; 022 private static final int MaxStringCacheLen = 12; 023 private static final int StringCacheSize = 512; 024 private String[] stringCache; // holds reused strings in this doc, to lessen garbage 025 private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations 026 027 static final int BufferSize = 1024 * 2; // visible for testing 028 static final int RefillPoint = BufferSize / 2; // when bufPos characters read, refill; visible for testing 029 private static final int RewindLimit = 1024; // the maximum we can rewind. No HTML entities can be larger than this. 030 031 private Reader reader; // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader 032 private char[] charBuf; // character buffer we consume from; filled from Reader 033 private int bufPos; // position in charBuf that's been consumed to 034 private int bufLength; // the num of characters actually buffered in charBuf, <= charBuf.length 035 private int fillPoint = 0; // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp 036 private int consumed; // how many characters total have been consumed from this CharacterReader (less the current bufPos) 037 private int bufMark = -1; // if not -1, the marked rewind position 038 private boolean readFully; // if the underlying stream has been completely read, no value in further buffering 039 040 private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer 041 042 @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions 043 044 public CharacterReader(Reader input, int sz) { 045 this(input); // sz is no longer used 046 } 047 048 public CharacterReader(Reader input) { 049 Validate.notNull(input); 050 reader = input; 051 charBuf = BufferPool.borrow(); 052 stringCache = StringPool.borrow(); 053 bufferUp(); 054 } 055 056 public CharacterReader(String input) { 057 this(new StringReader(input)); 058 } 059 060 @Override 061 public void close() { 062 if (reader == null) 063 return; 064 try { 065 reader.close(); 066 } catch (IOException ignored) { 067 } finally { 068 reader = null; 069 Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer 070 BufferPool.release(charBuf); 071 charBuf = null; 072 StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents 073 stringCache = null; 074 lineMap = null; 075 } 076 } 077 078 private void bufferUp() { 079 if (readFully || bufPos < fillPoint || bufMark != -1) 080 return; 081 doBufferUp(); // structured so bufferUp may become an intrinsic candidate 082 } 083 084 /** 085 Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException. 086 @throws UncheckedIOException if the underlying reader throws an IOException 087 */ 088 private void doBufferUp() { 089 /* 090 The flow: 091 - if read fully, or if bufPos < fillPoint, or if marked - do not fill. 092 - update readerPos (total amount consumed from this CharacterReader) += bufPos 093 - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount 094 - loop read the Reader until we fill charBuf. bufLength += read. 095 - readFully = true when read = -1 096 */ 097 consumed += bufPos; 098 bufLength -= bufPos; 099 if (bufLength > 0) 100 System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength); 101 bufPos = 0; 102 while (bufLength < BufferSize) { 103 try { 104 int read = reader.read(charBuf, bufLength, charBuf.length - bufLength); 105 if (read == -1) { 106 readFully = true; 107 break; 108 } 109 if (read == 0) { 110 break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed 111 } 112 bufLength += read; 113 } catch (IOException e) { 114 throw new UncheckedIOException(e); 115 } 116 } 117 fillPoint = Math.min(bufLength, RefillPoint); 118 119 scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking 120 } 121 122 void mark() { 123 // make sure there is enough look ahead capacity 124 if (bufLength - bufPos < RewindLimit) 125 fillPoint = 0; 126 127 bufferUp(); 128 bufMark = bufPos; 129 } 130 131 void unmark() { 132 bufMark = -1; 133 } 134 135 void rewindToMark() { 136 if (bufMark == -1) 137 throw new UncheckedIOException(new IOException("Mark invalid")); 138 139 bufPos = bufMark; 140 unmark(); 141 } 142 143 /** 144 * Gets the position currently read to in the content. Starts at 0. 145 * @return current position 146 */ 147 public int pos() { 148 // consuming EOF advances to a virtual position so it can be unconsumed; don't expose that beyond the input 149 return consumed + Math.min(bufPos, bufLength); 150 } 151 152 /** 153 Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the 154 legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of 155 use. 156 157 @param track set tracking on|off 158 @since 1.14.3 159 */ 160 public void trackNewlines(boolean track) { 161 if (track && lineMap == null) { 162 lineMap = new LineMap(); 163 scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp 164 } 165 else if (!track) 166 lineMap = null; 167 } 168 169 /** 170 Check if the tracking of newlines is enabled. 171 @return the current newline tracking state 172 @since 1.14.3 173 */ 174 public boolean isTrackNewlines() { 175 return lineMap != null; 176 } 177 178 /** 179 Get the line map enabled by {@link #trackNewlines(boolean)}. 180 */ 181 LineMap lineMap() { 182 assert lineMap != null; 183 return lineMap; 184 } 185 186 /** 187 Get the current line number (that the reader has consumed to). Starts at line #1. 188 @return the current line number, or 1 if line tracking is not enabled. 189 @since 1.14.3 190 @see #trackNewlines(boolean) 191 */ 192 public int lineNumber() { 193 return lineNumber(pos()); 194 } 195 196 int lineNumber(int pos) { 197 if (!isTrackNewlines()) 198 return 1; 199 200 return lineMap().lineNumber(pos); 201 } 202 203 /** 204 Get the current column number (that the reader has consumed to). Starts at column #1. 205 @return the current column number 206 @since 1.14.3 207 @see #trackNewlines(boolean) 208 */ 209 public int columnNumber() { 210 return columnNumber(pos()); 211 } 212 213 int columnNumber(int pos) { 214 if (!isTrackNewlines()) 215 return pos + 1; 216 217 return lineMap().columnNumber(pos); 218 } 219 220 /** 221 Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line 222 number 5 and column number 10. 223 @return line:col position 224 @since 1.14.3 225 @see #trackNewlines(boolean) 226 */ 227 String posLineCol() { 228 return lineNumber() + ":" + columnNumber(); 229 } 230 231 /** 232 Scans the buffer for newline positions and records line starts. 233 */ 234 private void scanBufferForNewlines() { 235 if (!isTrackNewlines()) 236 return; 237 238 for (int i = bufPos; i < bufLength; i++) { 239 if (charBuf[i] == '\n') { 240 int lineStart = 1 + consumed + i; 241 lineMap().addLineStart(lineStart); 242 } 243 } 244 } 245 246 /** 247 * Tests if all the content has been read. 248 * @return true if nothing left to read. 249 */ 250 public boolean isEmpty() { 251 bufferUp(); 252 return bufPos >= bufLength; 253 } 254 255 private boolean isEmptyNoBufferUp() { 256 return bufPos >= bufLength; 257 } 258 259 /** 260 * Get the char at the current position. 261 * @return char 262 */ 263 public char current() { 264 bufferUp(); 265 return isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 266 } 267 268 /** 269 Consume one character off the queue. 270 @return first character on queue, or EOF if the queue is empty. 271 */ 272 public char consume() { 273 bufferUp(); 274 char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos]; 275 bufPos++; 276 return val; 277 } 278 279 /** 280 Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp. 281 */ 282 void unconsume() { 283 if (bufPos < 1) 284 throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it. 285 286 bufPos--; 287 } 288 289 /** 290 * Moves the current position by one. 291 */ 292 public void advance() { 293 bufPos++; 294 } 295 296 /** 297 * Returns the number of characters between the current position and the next instance of the input char 298 * @param c scan target 299 * @return offset between current position and next instance of target. -1 if not found. 300 */ 301 int nextIndexOf(char c) { 302 // doesn't handle scanning for surrogates 303 bufferUp(); 304 for (int i = bufPos; i < bufLength; i++) { 305 if (c == charBuf[i]) 306 return i - bufPos; 307 } 308 return -1; 309 } 310 311 /** 312 * Returns the number of characters between the current position and the next instance of the input sequence 313 * 314 * @param seq scan target 315 * @return offset between current position and next instance of target. -1 if not found. 316 */ 317 int nextIndexOf(CharSequence seq) { 318 bufferUp(); 319 // doesn't handle scanning for surrogates 320 char startChar = seq.charAt(0); 321 for (int offset = bufPos; offset < bufLength; offset++) { 322 // scan to first instance of startchar: 323 if (startChar != charBuf[offset]) 324 while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ } 325 int i = offset + 1; 326 int last = i + seq.length()-1; 327 if (offset < bufLength && last <= bufLength) { 328 for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ } 329 if (i == last) // found full sequence 330 return offset - bufPos; 331 } 332 } 333 return -1; 334 } 335 336 /** 337 * Reads characters up to the specific char. 338 * @param c the delimiter 339 * @return the chars read 340 */ 341 public String consumeTo(char c) { 342 int offset = nextIndexOf(c); 343 if (offset != -1) { 344 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 345 bufPos += offset; 346 return consumed; 347 } else { 348 return consumeToEnd(); 349 } 350 } 351 352 /** 353 Reads the characters up to (but not including) the specified case-sensitive string. 354 <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the 355 length of the sequence, such that this call may be repeated. 356 @param seq the delimiter 357 @return the chars read 358 */ 359 public String consumeTo(String seq) { 360 int offset = nextIndexOf(seq); 361 if (offset != -1) { 362 String consumed = cacheString(charBuf, stringCache, bufPos, offset); 363 bufPos += offset; 364 return consumed; 365 } else if (bufLength - bufPos < seq.length()) { 366 // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF 367 return consumeToEnd(); 368 } else { 369 // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters 370 // unread in case they contain the beginning of the search string 371 int endPos = bufLength - seq.length() + 1; 372 String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos); 373 bufPos = endPos; 374 return consumed; 375 } 376 } 377 378 /** 379 Read characters while the input predicate returns true. 380 @return characters read 381 */ 382 String consumeMatching(CharPredicate func) { 383 return consumeMatching(func, -1); 384 } 385 386 /** 387 Read characters while the input predicate returns true, up to a maximum length. 388 @param func predicate to test 389 @param maxLength maximum length to read. -1 indicates no maximum 390 @return characters read 391 */ 392 String consumeMatching(CharPredicate func, int maxLength) { 393 bufferUp(); 394 int pos = bufPos; 395 final int start = pos; 396 final int remaining = bufLength; 397 final char[] val = charBuf; 398 399 while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) { 400 pos++; 401 } 402 403 bufPos = pos; 404 return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : ""; 405 } 406 407 /** 408 Read characters until the first of any delimiters is found. 409 @param chars delimiters to scan for 410 @return characters read up to the matched delimiter. 411 */ 412 public String consumeToAny(final char... chars) { 413 bufferUp(); 414 int pos = bufPos; 415 final int start = pos; 416 final int remaining = bufLength; 417 final char[] val = charBuf; 418 419 scan: 420 while (pos < remaining) { 421 char c = val[pos]; 422 for (char seek : chars) 423 if (c == seek) break scan; 424 pos++; 425 } 426 427 return consumeRange(start, pos); 428 } 429 430 /** 431 Read characters until either delimiter is found. 432 */ 433 String consumeToAny(char c1, char c2) { 434 // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func) 435 bufferUp(); 436 int pos = bufPos; 437 final int start = pos; 438 final int remaining = bufLength; 439 final char[] val = charBuf; 440 441 while (pos < remaining) { 442 char c = val[pos]; 443 if (c == c1 || c == c2) break; 444 pos++; 445 } 446 447 return consumeRange(start, pos); 448 } 449 450 /** 451 Read characters until any delimiter is found. 452 */ 453 String consumeToAny(char c1, char c2, char c3) { 454 bufferUp(); 455 int pos = bufPos; 456 final int start = pos; 457 final int remaining = bufLength; 458 final char[] val = charBuf; 459 460 while (pos < remaining) { 461 char c = val[pos]; 462 if (c == c1 || c == c2 || c == c3) break; 463 pos++; 464 } 465 466 return consumeRange(start, pos); 467 } 468 469 String consumeToAnySorted(final char... chars) { 470 bufferUp(); 471 int pos = bufPos; 472 final int start = pos; 473 final int remaining = bufLength; 474 final char[] val = charBuf; 475 476 while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) { 477 pos++; 478 } 479 480 return consumeRange(start, pos); 481 } 482 483 String consumeData() { 484 // consumes until &, <, null 485 return consumeToAny('&', '<', TokeniserState.nullChar); 486 } 487 488 String consumeAttributeQuoted(final boolean single) { 489 // null, " or ', & 490 char quote = single ? '\'' : '"'; 491 return consumeToAny(TokeniserState.nullChar, '&', quote); 492 } 493 494 String consumeRawData() { 495 // <, null 496 return consumeToAny('<', TokeniserState.nullChar); 497 } 498 499 String consumeTagName() { 500 // '\t', '\n', '\r', '\f', ' ', '/', '>' 501 // NOTE: out of spec; does not stop and append on nullChar but eats 502 bufferUp(); 503 int pos = bufPos; 504 final int start = pos; 505 final int remaining = bufLength; 506 final char[] val = charBuf; 507 508 while (pos < remaining) { 509 char c = val[pos]; 510 switch (c) { 511 case '\t': 512 case '\n': 513 case '\r': 514 case '\f': 515 case ' ': 516 case '/': 517 case '>': 518 return consumeRange(start, pos); 519 } 520 pos++; 521 } 522 523 return consumeRange(start, pos); 524 } 525 526 String consumeToEnd() { 527 bufferUp(); 528 String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos); 529 bufPos = bufLength; 530 return data; 531 } 532 533 /** Consumes ASCII letters used by the HTML tokenizer's name states. */ 534 String consumeLetterSequence() { 535 return consumeMatching(StringUtil::isAsciiLetter); 536 } 537 538 String consumeLetterThenDigitSequence() { 539 bufferUp(); 540 int start = bufPos; 541 while (bufPos < bufLength) { 542 if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++; 543 else break; 544 } 545 while (!isEmptyNoBufferUp()) { 546 if (StringUtil.isDigit(charBuf[bufPos])) bufPos++; 547 else break; 548 } 549 550 return cacheString(charBuf, stringCache, start, bufPos - start); 551 } 552 553 String consumeHexSequence() { 554 return consumeMatching(StringUtil::isHexDigit); 555 } 556 557 String consumeDigitSequence() { 558 return consumeMatching(c -> c >= '0' && c <= '9'); 559 } 560 561 /** 562 Complete a scan by moving the reader and returning the matched range. 563 */ 564 private String consumeRange(int start, int pos) { 565 bufPos = pos; 566 return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : ""; 567 } 568 569 boolean matches(char c) { 570 return !isEmpty() && charBuf[bufPos] == c; 571 } 572 573 boolean matches(String seq) { 574 bufferUp(); 575 int scanLength = seq.length(); 576 if (scanLength > bufLength - bufPos) 577 return false; 578 579 for (int offset = 0; offset < scanLength; offset++) 580 if (seq.charAt(offset) != charBuf[bufPos +offset]) 581 return false; 582 return true; 583 } 584 585 /** 586 Checks if the current buffer position matches the sequence case-insensitively. 587 */ 588 boolean matchesIgnoreCase(String seq) { 589 bufferUp(); 590 int scanLength = seq.length(); 591 if (scanLength > bufLength - bufPos) 592 return false; 593 594 return rangeMatchesIgnoreCase(seq, bufPos); 595 } 596 597 private boolean rangeMatchesIgnoreCase(String seq, int start) { 598 for (int offset = 0; offset < seq.length(); offset++) { 599 char scan = seq.charAt(offset); 600 char target = charBuf[start + offset]; 601 if (scan == target) continue; 602 603 scan = Character.toUpperCase(scan); 604 target = Character.toUpperCase(target); 605 if (scan != target) return false; 606 } 607 return true; 608 } 609 610 /** 611 Tests if the next character in the queue matches any of the characters in the sequence, case sensitively. 612 @param seq list of characters to check for 613 @return true if any matched, false if none did 614 */ 615 boolean matchesAny(char... seq) { 616 if (isEmpty()) 617 return false; 618 619 bufferUp(); 620 char c = charBuf[bufPos]; 621 for (char seek : seq) { 622 if (seek == c) 623 return true; 624 } 625 return false; 626 } 627 628 boolean matchesAnySorted(char[] seq) { 629 bufferUp(); 630 return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0; 631 } 632 633 /** 634 Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha 635 @return if it matches or not 636 */ 637 boolean matchesAsciiAlpha() { 638 if (isEmpty()) return false; 639 return StringUtil.isAsciiLetter(charBuf[bufPos]); 640 } 641 642 boolean matchesDigit() { 643 if (isEmpty()) return false; 644 return StringUtil.isDigit(charBuf[bufPos]); 645 } 646 647 boolean matchConsume(String seq) { 648 bufferUp(); 649 if (matches(seq)) { 650 bufPos += seq.length(); 651 return true; 652 } else { 653 return false; 654 } 655 } 656 657 boolean matchConsumeIgnoreCase(String seq) { 658 if (matchesIgnoreCase(seq)) { 659 bufPos += seq.length(); 660 return true; 661 } else { 662 return false; 663 } 664 } 665 666 @Override 667 public String toString() { 668 if (bufLength - bufPos < 0) return ""; 669 return new String(charBuf, bufPos, bufLength - bufPos); 670 } 671 672 /** 673 * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks. 674 * <p /> 675 * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list. 676 * That saves both having to create objects as hash keys, and running through the entry list, at the expense of 677 * some more duplicates. 678 */ 679 private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) { 680 if (count > MaxStringCacheLen) // don't cache strings that are too big 681 return new String(charBuf, start, count); 682 if (count < 1) 683 return ""; 684 685 // calculate hash: 686 int hash = 0; 687 int end = count + start; 688 for (int i = start; i < end; i++) { 689 hash = 31 * hash + charBuf[i]; 690 } 691 692 // get from cache 693 final int index = hash & StringCacheSize - 1; 694 String cached = stringCache[index]; 695 696 if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit 697 return cached; 698 else { 699 cached = new String(charBuf, start, count); 700 stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next 701 } 702 703 return cached; 704 } 705 706 /** 707 * Check if the value of the provided range equals the string. 708 */ 709 static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) { 710 if (count == cached.length()) { 711 int i = start; 712 int j = 0; 713 while (count-- != 0) { 714 if (charBuf[i++] != cached.charAt(j++)) 715 return false; 716 } 717 return true; 718 } 719 return false; 720 } 721 722 // just used for testing 723 boolean rangeEquals(final int start, final int count, final String cached) { 724 return rangeEquals(charBuf, start, count, cached); 725 } 726 727 @FunctionalInterface 728 interface CharPredicate { 729 boolean test(char c); 730 } 731}