001package org.jsoup.parser;
002
003import org.jsoup.helper.Validate;
004import org.jsoup.internal.LineMap;
005import org.jsoup.internal.SoftPool;
006import org.jsoup.internal.StringUtil;
007import org.jspecify.annotations.Nullable;
008
009import java.io.IOException;
010import java.io.UncheckedIOException;
011import java.io.Reader;
012import java.io.StringReader;
013import java.util.Arrays;
014
015/**
016 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes.
017 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an
018 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p>
019 */
020public final class CharacterReader implements AutoCloseable {
021    static final char EOF = (char) -1;
022    private static final int MaxStringCacheLen = 12;
023    private static final int StringCacheSize = 512;
024    private String[] stringCache; // holds reused strings in this doc, to lessen garbage
025    private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations
026
027    static final int BufferSize = 1024 * 2;         // visible for testing
028    static final int RefillPoint = BufferSize / 2;  // when bufPos characters read, refill; visible for testing
029    private static final int RewindLimit = 1024;    // the maximum we can rewind. No HTML entities can be larger than this.
030
031    private Reader reader;      // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader
032    private char[] charBuf;     // character buffer we consume from; filled from Reader
033    private int bufPos;         // position in charBuf that's been consumed to
034    private int bufLength;      // the num of characters actually buffered in charBuf, <= charBuf.length
035    private int fillPoint = 0;  // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp
036    private int consumed;       // how many characters total have been consumed from this CharacterReader (less the current bufPos)
037    private int bufMark = -1;   // if not -1, the marked rewind position
038    private boolean readFully;  // if the underlying stream has been completely read, no value in further buffering
039
040    private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer
041
042    @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions
043
044    public CharacterReader(Reader input, int sz) {
045        this(input); // sz is no longer used
046    }
047
048    public CharacterReader(Reader input) {
049        Validate.notNull(input);
050        reader = input;
051        charBuf = BufferPool.borrow();
052        stringCache = StringPool.borrow();
053        bufferUp();
054    }
055
056    public CharacterReader(String input) {
057        this(new StringReader(input));
058    }
059
060    @Override
061    public void close() {
062        if (reader == null)
063            return;
064        try {
065            reader.close();
066        } catch (IOException ignored) {
067        } finally {
068            reader = null;
069            Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer
070            BufferPool.release(charBuf);
071            charBuf = null;
072            StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents
073            stringCache = null;
074            lineMap = null;
075        }
076    }
077
078    private void bufferUp() {
079        if (readFully || bufPos < fillPoint || bufMark != -1)
080            return;
081        doBufferUp(); // structured so bufferUp may become an intrinsic candidate
082    }
083
084    /**
085     Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException.
086     @throws UncheckedIOException if the underlying reader throws an IOException
087     */
088    private void doBufferUp() {
089        /*
090        The flow:
091        - if read fully, or if bufPos < fillPoint, or if marked - do not fill.
092        - update readerPos (total amount consumed from this CharacterReader) += bufPos
093        - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount
094        - loop read the Reader until we fill charBuf. bufLength += read.
095        - readFully = true when read = -1
096         */
097        consumed += bufPos;
098        bufLength -= bufPos;
099        if (bufLength > 0)
100            System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength);
101        bufPos = 0;
102        while (bufLength < BufferSize) {
103            try {
104                int read = reader.read(charBuf, bufLength, charBuf.length - bufLength);
105                if (read == -1) {
106                    readFully = true;
107                    break;
108                }
109                if (read == 0) {
110                    break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed
111                }
112                bufLength += read;
113            } catch (IOException e) {
114                throw new UncheckedIOException(e);
115            }
116        }
117        fillPoint = Math.min(bufLength, RefillPoint);
118
119        scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking
120    }
121
122    void mark() {
123        // make sure there is enough look ahead capacity
124        if (bufLength - bufPos < RewindLimit)
125            fillPoint = 0;
126
127        bufferUp();
128        bufMark = bufPos;
129    }
130
131    void unmark() {
132        bufMark = -1;
133    }
134
135    void rewindToMark() {
136        if (bufMark == -1)
137            throw new UncheckedIOException(new IOException("Mark invalid"));
138
139        bufPos = bufMark;
140        unmark();
141    }
142
143    /**
144     * Gets the position currently read to in the content. Starts at 0.
145     * @return current position
146     */
147    public int pos() {
148        // consuming EOF advances to a virtual position so it can be unconsumed; don't expose that beyond the input
149        return consumed + Math.min(bufPos, bufLength);
150    }
151
152    /**
153     Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the
154     legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of
155     use.
156
157     @param track set tracking on|off
158     @since 1.14.3
159     */
160    public void trackNewlines(boolean track) {
161        if (track && lineMap == null) {
162            lineMap = new LineMap();
163            scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp
164        }
165        else if (!track)
166            lineMap = null;
167    }
168
169    /**
170     Check if the tracking of newlines is enabled.
171     @return the current newline tracking state
172     @since 1.14.3
173     */
174    public boolean isTrackNewlines() {
175        return lineMap != null;
176    }
177
178    /**
179     Get the line map enabled by {@link #trackNewlines(boolean)}.
180     */
181    LineMap lineMap() {
182        assert lineMap != null;
183        return lineMap;
184    }
185
186    /**
187     Get the current line number (that the reader has consumed to). Starts at line #1.
188     @return the current line number, or 1 if line tracking is not enabled.
189     @since 1.14.3
190     @see #trackNewlines(boolean)
191     */
192    public int lineNumber() {
193        return lineNumber(pos());
194    }
195
196    int lineNumber(int pos) {
197        if (!isTrackNewlines())
198            return 1;
199
200        return lineMap().lineNumber(pos);
201    }
202
203    /**
204     Get the current column number (that the reader has consumed to). Starts at column #1.
205     @return the current column number
206     @since 1.14.3
207     @see #trackNewlines(boolean)
208     */
209    public int columnNumber() {
210        return columnNumber(pos());
211    }
212
213    int columnNumber(int pos) {
214        if (!isTrackNewlines())
215            return pos + 1;
216
217        return lineMap().columnNumber(pos);
218    }
219
220    /**
221     Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line
222     number 5 and column number 10.
223     @return line:col position
224     @since 1.14.3
225     @see #trackNewlines(boolean)
226     */
227    String posLineCol() {
228        return lineNumber() + ":" + columnNumber();
229    }
230
231    /**
232     Scans the buffer for newline positions and records line starts.
233     */
234    private void scanBufferForNewlines() {
235        if (!isTrackNewlines())
236            return;
237
238        for (int i = bufPos; i < bufLength; i++) {
239            if (charBuf[i] == '\n') {
240                int lineStart = 1 + consumed + i;
241                lineMap().addLineStart(lineStart);
242            }
243        }
244    }
245
246    /**
247     * Tests if all the content has been read.
248     * @return true if nothing left to read.
249     */
250    public boolean isEmpty() {
251        bufferUp();
252        return bufPos >= bufLength;
253    }
254
255    private boolean isEmptyNoBufferUp() {
256        return bufPos >= bufLength;
257    }
258
259    /**
260     * Get the char at the current position.
261     * @return char
262     */
263    public char current() {
264        bufferUp();
265        return isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
266    }
267
268    /**
269     Consume one character off the queue.
270     @return first character on queue, or EOF if the queue is empty.
271     */
272    public char consume() {
273        bufferUp();
274        char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
275        bufPos++;
276        return val;
277    }
278
279    /**
280     Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp.
281     */
282    void unconsume() {
283        if (bufPos < 1)
284            throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it.
285
286        bufPos--;
287    }
288
289    /**
290     * Moves the current position by one.
291     */
292    public void advance() {
293        bufPos++;
294    }
295
296    /**
297     * Returns the number of characters between the current position and the next instance of the input char
298     * @param c scan target
299     * @return offset between current position and next instance of target. -1 if not found.
300     */
301    int nextIndexOf(char c) {
302        // doesn't handle scanning for surrogates
303        bufferUp();
304        for (int i = bufPos; i < bufLength; i++) {
305            if (c == charBuf[i])
306                return i - bufPos;
307        }
308        return -1;
309    }
310
311    /**
312     * Returns the number of characters between the current position and the next instance of the input sequence
313     *
314     * @param seq scan target
315     * @return offset between current position and next instance of target. -1 if not found.
316     */
317    int nextIndexOf(CharSequence seq) {
318        bufferUp();
319        // doesn't handle scanning for surrogates
320        char startChar = seq.charAt(0);
321        for (int offset = bufPos; offset < bufLength; offset++) {
322            // scan to first instance of startchar:
323            if (startChar != charBuf[offset])
324                while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ }
325            int i = offset + 1;
326            int last = i + seq.length()-1;
327            if (offset < bufLength && last <= bufLength) {
328                for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ }
329                if (i == last) // found full sequence
330                    return offset - bufPos;
331            }
332        }
333        return -1;
334    }
335
336    /**
337     * Reads characters up to the specific char.
338     * @param c the delimiter
339     * @return the chars read
340     */
341    public String consumeTo(char c) {
342        int offset = nextIndexOf(c);
343        if (offset != -1) {
344            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
345            bufPos += offset;
346            return consumed;
347        } else {
348            return consumeToEnd();
349        }
350    }
351
352    /**
353     Reads the characters up to (but not including) the specified case-sensitive string.
354     <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the
355     length of the sequence, such that this call may be repeated.
356     @param seq the delimiter
357     @return the chars read
358     */
359    public String consumeTo(String seq) {
360        int offset = nextIndexOf(seq);
361        if (offset != -1) {
362            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
363            bufPos += offset;
364            return consumed;
365        } else if (bufLength - bufPos < seq.length()) {
366            // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF
367            return consumeToEnd();
368        } else {
369            // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters
370            // unread in case they contain the beginning of the search string
371            int endPos = bufLength - seq.length() + 1;
372            String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos);
373            bufPos = endPos;
374            return consumed;
375        }
376    }
377
378    /**
379     Read characters while the input predicate returns true.
380     @return characters read
381     */
382    String consumeMatching(CharPredicate func) {
383        return consumeMatching(func, -1);
384    }
385
386    /**
387     Read characters while the input predicate returns true, up to a maximum length.
388     @param func predicate to test
389     @param maxLength maximum length to read. -1 indicates no maximum
390     @return characters read
391     */
392    String consumeMatching(CharPredicate func, int maxLength) {
393        bufferUp();
394        int pos = bufPos;
395        final int start = pos;
396        final int remaining = bufLength;
397        final char[] val = charBuf;
398
399        while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) {
400            pos++;
401        }
402
403        bufPos = pos;
404        return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : "";
405    }
406
407    /**
408     Read characters until the first of any delimiters is found.
409     @param chars delimiters to scan for
410     @return characters read up to the matched delimiter.
411     */
412    public String consumeToAny(final char... chars) {
413        bufferUp();
414        int pos = bufPos;
415        final int start = pos;
416        final int remaining = bufLength;
417        final char[] val = charBuf;
418
419        scan:
420        while (pos < remaining) {
421            char c = val[pos];
422            for (char seek : chars)
423                if (c == seek) break scan;
424            pos++;
425        }
426
427        return consumeRange(start, pos);
428    }
429
430    /**
431     Read characters until either delimiter is found.
432     */
433    String consumeToAny(char c1, char c2) {
434        // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func)
435        bufferUp();
436        int pos = bufPos;
437        final int start = pos;
438        final int remaining = bufLength;
439        final char[] val = charBuf;
440
441        while (pos < remaining) {
442            char c = val[pos];
443            if (c == c1 || c == c2) break;
444            pos++;
445        }
446
447        return consumeRange(start, pos);
448    }
449
450    /**
451     Read characters until any delimiter is found.
452     */
453    String consumeToAny(char c1, char c2, char c3) {
454        bufferUp();
455        int pos = bufPos;
456        final int start = pos;
457        final int remaining = bufLength;
458        final char[] val = charBuf;
459
460        while (pos < remaining) {
461            char c = val[pos];
462            if (c == c1 || c == c2 || c == c3) break;
463            pos++;
464        }
465
466        return consumeRange(start, pos);
467    }
468
469    String consumeToAnySorted(final char... chars) {
470        bufferUp();
471        int pos = bufPos;
472        final int start = pos;
473        final int remaining = bufLength;
474        final char[] val = charBuf;
475
476        while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) {
477            pos++;
478        }
479
480        return consumeRange(start, pos);
481    }
482
483    String consumeData() {
484        // consumes until &, <, null
485        return consumeToAny('&', '<', TokeniserState.nullChar);
486    }
487
488    String consumeAttributeQuoted(final boolean single) {
489        // null, " or ', &
490        char quote = single ? '\'' : '"';
491        return consumeToAny(TokeniserState.nullChar, '&', quote);
492    }
493
494    String consumeRawData() {
495        // <, null
496        return consumeToAny('<', TokeniserState.nullChar);
497    }
498
499    String consumeTagName() {
500        // '\t', '\n', '\r', '\f', ' ', '/', '>'
501        // NOTE: out of spec; does not stop and append on nullChar but eats
502        bufferUp();
503        int pos = bufPos;
504        final int start = pos;
505        final int remaining = bufLength;
506        final char[] val = charBuf;
507
508        while (pos < remaining) {
509            char c = val[pos];
510            switch (c) {
511                case '\t':
512                case '\n':
513                case '\r':
514                case '\f':
515                case ' ':
516                case '/':
517                case '>':
518                    return consumeRange(start, pos);
519            }
520            pos++;
521        }
522
523        return consumeRange(start, pos);
524    }
525
526    String consumeToEnd() {
527        bufferUp();
528        String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos);
529        bufPos = bufLength;
530        return data;
531    }
532
533    /** Consumes ASCII letters used by the HTML tokenizer's name states. */
534    String consumeLetterSequence() {
535        return consumeMatching(StringUtil::isAsciiLetter);
536    }
537
538    String consumeLetterThenDigitSequence() {
539        bufferUp();
540        int start = bufPos;
541        while (bufPos < bufLength) {
542            if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++;
543            else break;
544        }
545        while (!isEmptyNoBufferUp()) {
546            if (StringUtil.isDigit(charBuf[bufPos])) bufPos++;
547            else break;
548        }
549
550        return cacheString(charBuf, stringCache, start, bufPos - start);
551    }
552
553    String consumeHexSequence() {
554        return consumeMatching(StringUtil::isHexDigit);
555    }
556
557    String consumeDigitSequence() {
558        return consumeMatching(c -> c >= '0' && c <= '9');
559    }
560
561    /**
562     Complete a scan by moving the reader and returning the matched range.
563     */
564    private String consumeRange(int start, int pos) {
565        bufPos = pos;
566        return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : "";
567    }
568
569    boolean matches(char c) {
570        return !isEmpty() && charBuf[bufPos] == c;
571    }
572
573    boolean matches(String seq) {
574        bufferUp();
575        int scanLength = seq.length();
576        if (scanLength > bufLength - bufPos)
577            return false;
578
579        for (int offset = 0; offset < scanLength; offset++)
580            if (seq.charAt(offset) != charBuf[bufPos +offset])
581                return false;
582        return true;
583    }
584
585    /**
586     Checks if the current buffer position matches the sequence case-insensitively.
587     */
588    boolean matchesIgnoreCase(String seq) {
589        bufferUp();
590        int scanLength = seq.length();
591        if (scanLength > bufLength - bufPos)
592            return false;
593
594        return rangeMatchesIgnoreCase(seq, bufPos);
595    }
596
597    private boolean rangeMatchesIgnoreCase(String seq, int start) {
598        for (int offset = 0; offset < seq.length(); offset++) {
599            char scan = seq.charAt(offset);
600            char target = charBuf[start + offset];
601            if (scan == target) continue;
602
603            scan = Character.toUpperCase(scan);
604            target = Character.toUpperCase(target);
605            if (scan != target) return false;
606        }
607        return true;
608    }
609
610    /**
611     Tests if the next character in the queue matches any of the characters in the sequence, case sensitively.
612     @param seq list of characters to check for
613     @return true if any matched, false if none did
614     */
615    boolean matchesAny(char... seq) {
616        if (isEmpty())
617            return false;
618
619        bufferUp();
620        char c = charBuf[bufPos];
621        for (char seek : seq) {
622            if (seek == c)
623                return true;
624        }
625        return false;
626    }
627
628    boolean matchesAnySorted(char[] seq) {
629        bufferUp();
630        return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0;
631    }
632
633    /**
634     Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha
635     @return if it matches or not
636     */
637    boolean matchesAsciiAlpha() {
638        if (isEmpty()) return false;
639        return StringUtil.isAsciiLetter(charBuf[bufPos]);
640    }
641
642    boolean matchesDigit() {
643        if (isEmpty()) return false;
644        return StringUtil.isDigit(charBuf[bufPos]);
645    }
646
647    boolean matchConsume(String seq) {
648        bufferUp();
649        if (matches(seq)) {
650            bufPos += seq.length();
651            return true;
652        } else {
653            return false;
654        }
655    }
656
657    boolean matchConsumeIgnoreCase(String seq) {
658        if (matchesIgnoreCase(seq)) {
659            bufPos += seq.length();
660            return true;
661        } else {
662            return false;
663        }
664    }
665
666    @Override
667    public String toString() {
668        if (bufLength - bufPos < 0) return "";
669        return new String(charBuf, bufPos, bufLength - bufPos);
670    }
671
672    /**
673     * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks.
674     * <p />
675     * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list.
676     * That saves both having to create objects as hash keys, and running through the entry list, at the expense of
677     * some more duplicates.
678     */
679    private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) {
680        if (count > MaxStringCacheLen) // don't cache strings that are too big
681            return new String(charBuf, start, count);
682        if (count < 1)
683            return "";
684
685        // calculate hash:
686        int hash = 0;
687        int end = count + start;
688        for (int i = start; i < end; i++) {
689            hash = 31 * hash + charBuf[i];
690        }
691
692        // get from cache
693        final int index = hash & StringCacheSize - 1;
694        String cached = stringCache[index];
695
696        if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit
697            return cached;
698        else {
699            cached = new String(charBuf, start, count);
700            stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next
701        }
702
703        return cached;
704    }
705
706    /**
707     * Check if the value of the provided range equals the string.
708     */
709    static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) {
710        if (count == cached.length()) {
711            int i = start;
712            int j = 0;
713            while (count-- != 0) {
714                if (charBuf[i++] != cached.charAt(j++))
715                    return false;
716            }
717            return true;
718        }
719        return false;
720    }
721
722    // just used for testing
723    boolean rangeEquals(final int start, final int count, final String cached) {
724        return rangeEquals(charBuf, start, count, cached);
725    }
726
727    @FunctionalInterface
728    interface CharPredicate {
729        boolean test(char c);
730    }
731}