001package org.jsoup.parser;
002
003import org.jsoup.helper.Validate;
004import org.jsoup.internal.LineMap;
005import org.jsoup.internal.SoftPool;
006import org.jsoup.internal.StringUtil;
007import org.jspecify.annotations.Nullable;
008
009import java.io.IOException;
010import java.io.UncheckedIOException;
011import java.io.Reader;
012import java.io.StringReader;
013import java.util.Arrays;
014
015/**
016 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes.
017 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an
018 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p>
019 */
020public final class CharacterReader implements AutoCloseable {
021    static final char EOF = (char) -1;
022    private static final int MaxStringCacheLen = 12;
023    private static final int StringCacheSize = 512;
024    private String[] stringCache; // holds reused strings in this doc, to lessen garbage
025    private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations
026
027    static final int BufferSize = 1024 * 2;         // visible for testing
028    static final int RefillPoint = BufferSize / 2;  // when bufPos characters read, refill; visible for testing
029    private static final int RewindLimit = 1024;    // the maximum we can rewind. No HTML entities can be larger than this.
030
031    private Reader reader;      // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader
032    private char[] charBuf;     // character buffer we consume from; filled from Reader
033    private int bufPos;         // position in charBuf that's been consumed to
034    private int bufLength;      // the num of characters actually buffered in charBuf, <= charBuf.length
035    private int fillPoint = 0;  // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp
036    private int consumed;       // how many characters total have been consumed from this CharacterReader (less the current bufPos)
037    private int bufMark = -1;   // if not -1, the marked rewind position
038    private boolean readFully;  // if the underlying stream has been completely read, no value in further buffering
039
040    private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer
041
042    @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions
043
044    public CharacterReader(Reader input, int sz) {
045        this(input); // sz is no longer used
046    }
047
048    public CharacterReader(Reader input) {
049        Validate.notNull(input);
050        reader = input;
051        charBuf = BufferPool.borrow();
052        stringCache = StringPool.borrow();
053        bufferUp();
054    }
055
056    public CharacterReader(String input) {
057        this(new StringReader(input));
058    }
059
060    @Override
061    public void close() {
062        if (reader == null)
063            return;
064        try {
065            reader.close();
066        } catch (IOException ignored) {
067        } finally {
068            reader = null;
069            Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer
070            BufferPool.release(charBuf);
071            charBuf = null;
072            StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents
073            stringCache = null;
074            lineMap = null;
075        }
076    }
077
078    private void bufferUp() {
079        if (readFully || bufPos < fillPoint || bufMark != -1)
080            return;
081        doBufferUp(); // structured so bufferUp may become an intrinsic candidate
082    }
083
084    /**
085     Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException.
086     @throws UncheckedIOException if the underlying reader throws an IOException
087     */
088    private void doBufferUp() {
089        /*
090        The flow:
091        - if read fully, or if bufPos < fillPoint, or if marked - do not fill.
092        - update readerPos (total amount consumed from this CharacterReader) += bufPos
093        - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount
094        - loop read the Reader until we fill charBuf. bufLength += read.
095        - readFully = true when read = -1
096         */
097        consumed += bufPos;
098        bufLength -= bufPos;
099        if (bufLength > 0)
100            System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength);
101        bufPos = 0;
102        while (bufLength < BufferSize) {
103            try {
104                int read = reader.read(charBuf, bufLength, charBuf.length - bufLength);
105                if (read == -1) {
106                    readFully = true;
107                    break;
108                }
109                if (read == 0) {
110                    break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed
111                }
112                bufLength += read;
113            } catch (IOException e) {
114                throw new UncheckedIOException(e);
115            }
116        }
117        fillPoint = Math.min(bufLength, RefillPoint);
118
119        scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking
120        lastIcSeq = null; // cache for last containsIgnoreCase(seq)
121    }
122
123    void mark() {
124        // make sure there is enough look ahead capacity
125        if (bufLength - bufPos < RewindLimit)
126            fillPoint = 0;
127
128        bufferUp();
129        bufMark = bufPos;
130    }
131
132    void unmark() {
133        bufMark = -1;
134    }
135
136    void rewindToMark() {
137        if (bufMark == -1)
138            throw new UncheckedIOException(new IOException("Mark invalid"));
139
140        bufPos = bufMark;
141        unmark();
142    }
143
144    /**
145     * Gets the position currently read to in the content. Starts at 0.
146     * @return current position
147     */
148    public int pos() {
149        return consumed + bufPos;
150    }
151
152    /** Tests if the buffer has been fully read. */
153    boolean readFully() {
154        return readFully;
155    }
156
157    /**
158     Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the
159     legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of
160     use.
161
162     @param track set tracking on|off
163     @since 1.14.3
164     */
165    public void trackNewlines(boolean track) {
166        if (track && lineMap == null) {
167            lineMap = new LineMap();
168            scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp
169        }
170        else if (!track)
171            lineMap = null;
172    }
173
174    /**
175     Check if the tracking of newlines is enabled.
176     @return the current newline tracking state
177     @since 1.14.3
178     */
179    public boolean isTrackNewlines() {
180        return lineMap != null;
181    }
182
183    /**
184     Get the line map enabled by {@link #trackNewlines(boolean)}.
185     */
186    LineMap lineMap() {
187        assert lineMap != null;
188        return lineMap;
189    }
190
191    /**
192     Get the current line number (that the reader has consumed to). Starts at line #1.
193     @return the current line number, or 1 if line tracking is not enabled.
194     @since 1.14.3
195     @see #trackNewlines(boolean)
196     */
197    public int lineNumber() {
198        return lineNumber(pos());
199    }
200
201    int lineNumber(int pos) {
202        if (!isTrackNewlines())
203            return 1;
204
205        return lineMap().lineNumber(pos);
206    }
207
208    /**
209     Get the current column number (that the reader has consumed to). Starts at column #1.
210     @return the current column number
211     @since 1.14.3
212     @see #trackNewlines(boolean)
213     */
214    public int columnNumber() {
215        return columnNumber(pos());
216    }
217
218    int columnNumber(int pos) {
219        if (!isTrackNewlines())
220            return pos + 1;
221
222        return lineMap().columnNumber(pos);
223    }
224
225    /**
226     Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line
227     number 5 and column number 10.
228     @return line:col position
229     @since 1.14.3
230     @see #trackNewlines(boolean)
231     */
232    String posLineCol() {
233        return lineNumber() + ":" + columnNumber();
234    }
235
236    /**
237     Scans the buffer for newline positions and records line starts.
238     */
239    private void scanBufferForNewlines() {
240        if (!isTrackNewlines())
241            return;
242
243        for (int i = bufPos; i < bufLength; i++) {
244            if (charBuf[i] == '\n') {
245                int lineStart = 1 + consumed + i;
246                lineMap().addLineStart(lineStart);
247            }
248        }
249    }
250
251    /**
252     * Tests if all the content has been read.
253     * @return true if nothing left to read.
254     */
255    public boolean isEmpty() {
256        bufferUp();
257        return bufPos >= bufLength;
258    }
259
260    private boolean isEmptyNoBufferUp() {
261        return bufPos >= bufLength;
262    }
263
264    /**
265     * Get the char at the current position.
266     * @return char
267     */
268    public char current() {
269        bufferUp();
270        return isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
271    }
272
273    /**
274     Consume one character off the queue.
275     @return first character on queue, or EOF if the queue is empty.
276     */
277    public char consume() {
278        bufferUp();
279        char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
280        bufPos++;
281        return val;
282    }
283
284    /**
285     Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp.
286     */
287    void unconsume() {
288        if (bufPos < 1)
289            throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it.
290
291        bufPos--;
292    }
293
294    /**
295     * Moves the current position by one.
296     */
297    public void advance() {
298        bufPos++;
299    }
300
301    /**
302     * Returns the number of characters between the current position and the next instance of the input char
303     * @param c scan target
304     * @return offset between current position and next instance of target. -1 if not found.
305     */
306    int nextIndexOf(char c) {
307        // doesn't handle scanning for surrogates
308        bufferUp();
309        for (int i = bufPos; i < bufLength; i++) {
310            if (c == charBuf[i])
311                return i - bufPos;
312        }
313        return -1;
314    }
315
316    /**
317     * Returns the number of characters between the current position and the next instance of the input sequence
318     *
319     * @param seq scan target
320     * @return offset between current position and next instance of target. -1 if not found.
321     */
322    int nextIndexOf(CharSequence seq) {
323        bufferUp();
324        // doesn't handle scanning for surrogates
325        char startChar = seq.charAt(0);
326        for (int offset = bufPos; offset < bufLength; offset++) {
327            // scan to first instance of startchar:
328            if (startChar != charBuf[offset])
329                while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ }
330            int i = offset + 1;
331            int last = i + seq.length()-1;
332            if (offset < bufLength && last <= bufLength) {
333                for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ }
334                if (i == last) // found full sequence
335                    return offset - bufPos;
336            }
337        }
338        return -1;
339    }
340
341    /**
342     * Reads characters up to the specific char.
343     * @param c the delimiter
344     * @return the chars read
345     */
346    public String consumeTo(char c) {
347        int offset = nextIndexOf(c);
348        if (offset != -1) {
349            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
350            bufPos += offset;
351            return consumed;
352        } else {
353            return consumeToEnd();
354        }
355    }
356
357    /**
358     Reads the characters up to (but not including) the specified case-sensitive string.
359     <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the
360     length of the sequence, such that this call may be repeated.
361     @param seq the delimiter
362     @return the chars read
363     */
364    public String consumeTo(String seq) {
365        int offset = nextIndexOf(seq);
366        if (offset != -1) {
367            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
368            bufPos += offset;
369            return consumed;
370        } else if (bufLength - bufPos < seq.length()) {
371            // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF
372            return consumeToEnd();
373        } else {
374            // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters
375            // unread in case they contain the beginning of the search string
376            int endPos = bufLength - seq.length() + 1;
377            String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos);
378            bufPos = endPos;
379            return consumed;
380        }
381    }
382
383    /**
384     Read characters while the input predicate returns true.
385     @return characters read
386     */
387    String consumeMatching(CharPredicate func) {
388        return consumeMatching(func, -1);
389    }
390
391    /**
392     Read characters while the input predicate returns true, up to a maximum length.
393     @param func predicate to test
394     @param maxLength maximum length to read. -1 indicates no maximum
395     @return characters read
396     */
397    String consumeMatching(CharPredicate func, int maxLength) {
398        bufferUp();
399        int pos = bufPos;
400        final int start = pos;
401        final int remaining = bufLength;
402        final char[] val = charBuf;
403
404        while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) {
405            pos++;
406        }
407
408        bufPos = pos;
409        return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : "";
410    }
411
412    /**
413     Read characters until the first of any delimiters is found.
414     @param chars delimiters to scan for
415     @return characters read up to the matched delimiter.
416     */
417    public String consumeToAny(final char... chars) {
418        bufferUp();
419        int pos = bufPos;
420        final int start = pos;
421        final int remaining = bufLength;
422        final char[] val = charBuf;
423
424        scan:
425        while (pos < remaining) {
426            char c = val[pos];
427            for (char seek : chars)
428                if (c == seek) break scan;
429            pos++;
430        }
431
432        return consumeRange(start, pos);
433    }
434
435    /**
436     Read characters until either delimiter is found.
437     */
438    String consumeToAny(char c1, char c2) {
439        // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func)
440        bufferUp();
441        int pos = bufPos;
442        final int start = pos;
443        final int remaining = bufLength;
444        final char[] val = charBuf;
445
446        while (pos < remaining) {
447            char c = val[pos];
448            if (c == c1 || c == c2) break;
449            pos++;
450        }
451
452        return consumeRange(start, pos);
453    }
454
455    /**
456     Read characters until any delimiter is found.
457     */
458    String consumeToAny(char c1, char c2, char c3) {
459        bufferUp();
460        int pos = bufPos;
461        final int start = pos;
462        final int remaining = bufLength;
463        final char[] val = charBuf;
464
465        while (pos < remaining) {
466            char c = val[pos];
467            if (c == c1 || c == c2 || c == c3) break;
468            pos++;
469        }
470
471        return consumeRange(start, pos);
472    }
473
474    String consumeToAnySorted(final char... chars) {
475        bufferUp();
476        int pos = bufPos;
477        final int start = pos;
478        final int remaining = bufLength;
479        final char[] val = charBuf;
480
481        while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) {
482            pos++;
483        }
484
485        return consumeRange(start, pos);
486    }
487
488    String consumeData() {
489        // consumes until &, <, null
490        return consumeToAny('&', '<', TokeniserState.nullChar);
491    }
492
493    String consumeAttributeQuoted(final boolean single) {
494        // null, " or ', &
495        char quote = single ? '\'' : '"';
496        return consumeToAny(TokeniserState.nullChar, '&', quote);
497    }
498
499    String consumeRawData() {
500        // <, null
501        return consumeToAny('<', TokeniserState.nullChar);
502    }
503
504    String consumeTagName() {
505        // '\t', '\n', '\r', '\f', ' ', '/', '>'
506        // NOTE: out of spec; does not stop and append on nullChar but eats
507        bufferUp();
508        int pos = bufPos;
509        final int start = pos;
510        final int remaining = bufLength;
511        final char[] val = charBuf;
512
513        while (pos < remaining) {
514            char c = val[pos];
515            switch (c) {
516                case '\t':
517                case '\n':
518                case '\r':
519                case '\f':
520                case ' ':
521                case '/':
522                case '>':
523                    return consumeRange(start, pos);
524            }
525            pos++;
526        }
527
528        return consumeRange(start, pos);
529    }
530
531    String consumeToEnd() {
532        bufferUp();
533        String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos);
534        bufPos = bufLength;
535        return data;
536    }
537
538    String consumeLetterSequence() {
539        return consumeMatching(Character::isLetter);
540    }
541
542    String consumeLetterThenDigitSequence() {
543        bufferUp();
544        int start = bufPos;
545        while (bufPos < bufLength) {
546            if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++;
547            else break;
548        }
549        while (!isEmptyNoBufferUp()) {
550            if (StringUtil.isDigit(charBuf[bufPos])) bufPos++;
551            else break;
552        }
553
554        return cacheString(charBuf, stringCache, start, bufPos - start);
555    }
556
557    String consumeHexSequence() {
558        return consumeMatching(StringUtil::isHexDigit);
559    }
560
561    String consumeDigitSequence() {
562        return consumeMatching(c -> c >= '0' && c <= '9');
563    }
564
565    /**
566     Complete a scan by moving the reader and returning the matched range.
567     */
568    private String consumeRange(int start, int pos) {
569        bufPos = pos;
570        return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : "";
571    }
572
573    boolean matches(char c) {
574        return !isEmpty() && charBuf[bufPos] == c;
575    }
576
577    boolean matches(String seq) {
578        bufferUp();
579        int scanLength = seq.length();
580        if (scanLength > bufLength - bufPos)
581            return false;
582
583        for (int offset = 0; offset < scanLength; offset++)
584            if (seq.charAt(offset) != charBuf[bufPos +offset])
585                return false;
586        return true;
587    }
588
589    /**
590     Checks if the current buffer position matches the sequence case-insensitively.
591     */
592    boolean matchesIgnoreCase(String seq) {
593        bufferUp();
594        int scanLength = seq.length();
595        if (scanLength > bufLength - bufPos)
596            return false;
597
598        return rangeMatchesIgnoreCase(seq, bufPos);
599    }
600
601    private boolean rangeMatchesIgnoreCase(String seq, int start) {
602        for (int offset = 0; offset < seq.length(); offset++) {
603            char scan = seq.charAt(offset);
604            char target = charBuf[start + offset];
605            if (scan == target) continue;
606
607            scan = Character.toUpperCase(scan);
608            target = Character.toUpperCase(target);
609            if (scan != target) return false;
610        }
611        return true;
612    }
613
614    /**
615     Tests if the next character in the queue matches any of the characters in the sequence, case sensitively.
616     @param seq list of characters to check for
617     @return true if any matched, false if none did
618     */
619    boolean matchesAny(char... seq) {
620        if (isEmpty())
621            return false;
622
623        bufferUp();
624        char c = charBuf[bufPos];
625        for (char seek : seq) {
626            if (seek == c)
627                return true;
628        }
629        return false;
630    }
631
632    boolean matchesAnySorted(char[] seq) {
633        bufferUp();
634        return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0;
635    }
636
637    /**
638     Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha
639     @return if it matches or not
640     */
641    boolean matchesAsciiAlpha() {
642        if (isEmpty()) return false;
643        return StringUtil.isAsciiLetter(charBuf[bufPos]);
644    }
645
646    boolean matchesDigit() {
647        if (isEmpty()) return false;
648        return StringUtil.isDigit(charBuf[bufPos]);
649    }
650
651    boolean matchConsume(String seq) {
652        bufferUp();
653        if (matches(seq)) {
654            bufPos += seq.length();
655            return true;
656        } else {
657            return false;
658        }
659    }
660
661    boolean matchConsumeIgnoreCase(String seq) {
662        if (matchesIgnoreCase(seq)) {
663            bufPos += seq.length();
664            return true;
665        } else {
666            return false;
667        }
668    }
669
670    // we maintain a cache of the previously scanned sequence, and return that if applicable on repeated scans.
671    // that improves the situation where there is a sequence of <p<p<p<p<p<p<p...</title> and we're bashing on the <p
672    // looking for the </title>. Resets in bufferUp()
673    @Nullable private String lastIcSeq; // scan cache
674    private int lastIcIndex; // nearest found indexOf
675
676    /** Used to check presence of </title>, </style> when we're in RCData and see a <xxx. */
677    boolean containsIgnoreCase(String seq) {
678        bufferUp();
679        if (seq.equals(lastIcSeq)) {
680            if (lastIcIndex == -1) return false;
681            if (lastIcIndex >= bufPos) return true;
682        }
683        lastIcSeq = seq;
684
685        int scanLength = seq.length();
686        int maxStart = bufLength - scanLength;
687        for (int scan = bufPos; scan <= maxStart; scan++) {
688            if (rangeMatchesIgnoreCase(seq, scan)) {
689                lastIcIndex = scan;
690                return true;
691            }
692        }
693
694        lastIcIndex = -1;
695        return false;
696    }
697
698    @Override
699    public String toString() {
700        if (bufLength - bufPos < 0) return "";
701        return new String(charBuf, bufPos, bufLength - bufPos);
702    }
703
704    /**
705     * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks.
706     * <p />
707     * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list.
708     * That saves both having to create objects as hash keys, and running through the entry list, at the expense of
709     * some more duplicates.
710     */
711    private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) {
712        if (count > MaxStringCacheLen) // don't cache strings that are too big
713            return new String(charBuf, start, count);
714        if (count < 1)
715            return "";
716
717        // calculate hash:
718        int hash = 0;
719        int end = count + start;
720        for (int i = start; i < end; i++) {
721            hash = 31 * hash + charBuf[i];
722        }
723
724        // get from cache
725        final int index = hash & StringCacheSize - 1;
726        String cached = stringCache[index];
727
728        if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit
729            return cached;
730        else {
731            cached = new String(charBuf, start, count);
732            stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next
733        }
734
735        return cached;
736    }
737
738    /**
739     * Check if the value of the provided range equals the string.
740     */
741    static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) {
742        if (count == cached.length()) {
743            int i = start;
744            int j = 0;
745            while (count-- != 0) {
746                if (charBuf[i++] != cached.charAt(j++))
747                    return false;
748            }
749            return true;
750        }
751        return false;
752    }
753
754    // just used for testing
755    boolean rangeEquals(final int start, final int count, final String cached) {
756        return rangeEquals(charBuf, start, count, cached);
757    }
758
759    @FunctionalInterface
760    interface CharPredicate {
761        boolean test(char c);
762    }
763}