001package org.jsoup.parser;
002
003import org.jsoup.helper.Validate;
004import org.jsoup.internal.LineMap;
005import org.jsoup.internal.SoftPool;
006import org.jsoup.internal.StringUtil;
007import org.jspecify.annotations.Nullable;
008
009import java.io.IOException;
010import java.io.UncheckedIOException;
011import java.io.Reader;
012import java.io.StringReader;
013import java.util.Arrays;
014
015import static org.jsoup.internal.Normalizer.asciiLowerCase;
016
017/**
018 CharacterReader consumes tokens off a string. Used internally by jsoup. API subject to changes.
019 <p>If the underlying reader throws an IOException during any operation, the CharacterReader will throw an
020 {@link UncheckedIOException}. That won't happen with String / StringReader inputs.</p>
021 */
022public final class CharacterReader implements AutoCloseable {
023    static final char EOF = (char) -1;
024    private static final int MaxStringCacheLen = 12;
025    private static final int StringCacheSize = 512;
026    private String[] stringCache; // holds reused strings in this doc, to lessen garbage
027    private static final SoftPool<String[]> StringPool = new SoftPool<>(() -> new String[StringCacheSize]); // reuse cache between iterations
028
029    static final int BufferSize = 1024 * 2;         // visible for testing
030    static final int RefillPoint = BufferSize / 2;  // when bufPos characters read, refill; visible for testing
031    private static final int RewindLimit = 1024;    // the maximum we can rewind. No HTML entities can be larger than this.
032
033    private Reader reader;      // underlying Reader, will be backed by a buffered+controlled input stream, or StringReader
034    private char[] charBuf;     // character buffer we consume from; filled from Reader
035    private int bufPos;         // position in charBuf that's been consumed to
036    private int bufLength;      // the num of characters actually buffered in charBuf, <= charBuf.length
037    private int fillPoint = 0;  // how far into the charBuf we read before re-filling. 0.5 of charBuf.length after bufferUp
038    private int consumed;       // how many characters total have been consumed from this CharacterReader (less the current bufPos)
039    private int bufMark = -1;   // if not -1, the marked rewind position
040    private boolean readFully;  // if the underlying stream has been completely read, no value in further buffering
041
042    private static final SoftPool<char[]> BufferPool = new SoftPool<>(() -> new char[BufferSize]); // recycled char buffer
043
044    @Nullable private LineMap lineMap = null; // optionally maps source offsets to line and column positions
045
046    public CharacterReader(Reader input, int sz) {
047        this(input); // sz is no longer used
048    }
049
050    public CharacterReader(Reader input) {
051        Validate.notNull(input);
052        reader = input;
053        charBuf = BufferPool.borrow();
054        stringCache = StringPool.borrow();
055        bufferUp();
056    }
057
058    public CharacterReader(String input) {
059        this(new StringReader(input));
060    }
061
062    @Override
063    public void close() {
064        if (reader == null)
065            return;
066        try {
067            reader.close();
068        } catch (IOException ignored) {
069        } finally {
070            reader = null;
071            Arrays.fill(charBuf, (char) 0); // before release, clear the buffer. Not required, but acts as a safety net, and makes debug view clearer
072            BufferPool.release(charBuf);
073            charBuf = null;
074            StringPool.release(stringCache); // conversely, we don't clear the string cache, so we can reuse the contents
075            stringCache = null;
076            lineMap = null;
077        }
078    }
079
080    private void bufferUp() {
081        if (readFully || bufPos < fillPoint || bufMark != -1)
082            return;
083        doBufferUp(); // structured so bufferUp may become an intrinsic candidate
084    }
085
086    /**
087     Reads into the buffer. Will throw an UncheckedIOException if the underling reader throws an IOException.
088     @throws UncheckedIOException if the underlying reader throws an IOException
089     */
090    private void doBufferUp() {
091        /*
092        The flow:
093        - if read fully, or if bufPos < fillPoint, or if marked - do not fill.
094        - update readerPos (total amount consumed from this CharacterReader) += bufPos
095        - shift charBuf contents such that bufPos = 0; set next read offset (bufLength) -= shift amount
096        - loop read the Reader until we fill charBuf. bufLength += read.
097        - readFully = true when read = -1
098         */
099        consumed += bufPos;
100        bufLength -= bufPos;
101        if (bufLength > 0)
102            System.arraycopy(charBuf, bufPos, charBuf, 0, bufLength);
103        bufPos = 0;
104        while (bufLength < BufferSize) {
105            try {
106                int read = reader.read(charBuf, bufLength, charBuf.length - bufLength);
107                if (read == -1) {
108                    readFully = true;
109                    break;
110                }
111                if (read == 0) {
112                    break; // if we have a surrogate on the buffer boundary and trying to read 1; will have enough in our buffer to proceed
113                }
114                bufLength += read;
115            } catch (IOException e) {
116                throw new UncheckedIOException(e);
117            }
118        }
119        fillPoint = Math.min(bufLength, RefillPoint);
120
121        scanBufferForNewlines(); // if enabled, we index newline positions for line number tracking
122    }
123
124    void mark() {
125        // make sure there is enough look ahead capacity
126        if (bufLength - bufPos < RewindLimit)
127            fillPoint = 0;
128
129        bufferUp();
130        bufMark = bufPos;
131    }
132
133    void unmark() {
134        bufMark = -1;
135    }
136
137    void rewindToMark() {
138        if (bufMark == -1)
139            throw new UncheckedIOException(new IOException("Mark invalid"));
140
141        bufPos = bufMark;
142        unmark();
143    }
144
145    /**
146     * Gets the position currently read to in the content. Starts at 0.
147     * @return current position
148     */
149    public int pos() {
150        // consuming EOF advances to a virtual position so it can be unconsumed; don't expose that beyond the input
151        return consumed + Math.min(bufPos, bufLength);
152    }
153
154    /**
155     Enables or disables line number tracking. By default, will be <b>off</b>.Tracking line numbers improves the
156     legibility of parser error messages, for example. Tracking should be enabled before any content is read to be of
157     use.
158
159     @param track set tracking on|off
160     @since 1.14.3
161     */
162    public void trackNewlines(boolean track) {
163        if (track && lineMap == null) {
164            lineMap = new LineMap();
165            scanBufferForNewlines(); // first pass when enabled; subsequently called during bufferUp
166        }
167        else if (!track)
168            lineMap = null;
169    }
170
171    /**
172     Check if the tracking of newlines is enabled.
173     @return the current newline tracking state
174     @since 1.14.3
175     */
176    public boolean isTrackNewlines() {
177        return lineMap != null;
178    }
179
180    /**
181     Get the line map enabled by {@link #trackNewlines(boolean)}.
182     */
183    LineMap lineMap() {
184        assert lineMap != null;
185        return lineMap;
186    }
187
188    /**
189     Get the current line number (that the reader has consumed to). Starts at line #1.
190     @return the current line number, or 1 if line tracking is not enabled.
191     @since 1.14.3
192     @see #trackNewlines(boolean)
193     */
194    public int lineNumber() {
195        return lineNumber(pos());
196    }
197
198    int lineNumber(int pos) {
199        if (!isTrackNewlines())
200            return 1;
201
202        return lineMap().lineNumber(pos);
203    }
204
205    /**
206     Get the current column number (that the reader has consumed to). Starts at column #1.
207     @return the current column number
208     @since 1.14.3
209     @see #trackNewlines(boolean)
210     */
211    public int columnNumber() {
212        return columnNumber(pos());
213    }
214
215    int columnNumber(int pos) {
216        if (!isTrackNewlines())
217            return pos + 1;
218
219        return lineMap().columnNumber(pos);
220    }
221
222    /**
223     Get a formatted string representing the current line and column positions. E.g. <code>5:10</code> indicating line
224     number 5 and column number 10.
225     @return line:col position
226     @since 1.14.3
227     @see #trackNewlines(boolean)
228     */
229    String posLineCol() {
230        return lineNumber() + ":" + columnNumber();
231    }
232
233    /**
234     Scans the buffer for newline positions and records line starts.
235     */
236    private void scanBufferForNewlines() {
237        if (!isTrackNewlines())
238            return;
239
240        for (int i = bufPos; i < bufLength; i++) {
241            if (charBuf[i] == '\n') {
242                int lineStart = 1 + consumed + i;
243                lineMap().addLineStart(lineStart);
244            }
245        }
246    }
247
248    /**
249     * Tests if all the content has been read.
250     * @return true if nothing left to read.
251     */
252    public boolean isEmpty() {
253        bufferUp();
254        return bufPos >= bufLength;
255    }
256
257    private boolean isEmptyNoBufferUp() {
258        return bufPos >= bufLength;
259    }
260
261    /**
262     * Get the char at the current position.
263     * @return char
264     */
265    public char current() {
266        bufferUp();
267        return isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
268    }
269
270    /**
271     Consume one character off the queue.
272     @return first character on queue, or EOF if the queue is empty.
273     */
274    public char consume() {
275        bufferUp();
276        char val = isEmptyNoBufferUp() ? EOF : charBuf[bufPos];
277        bufPos++;
278        return val;
279    }
280
281    /**
282     Unconsume one character (bufPos--). MUST only be called directly after a consume(), and no chance of a bufferUp.
283     */
284    void unconsume() {
285        if (bufPos < 1)
286            throw new UncheckedIOException(new IOException("WTF: No buffer left to unconsume.")); // a bug if this fires, need to trace it.
287
288        bufPos--;
289    }
290
291    /**
292     * Moves the current position by one.
293     */
294    public void advance() {
295        bufPos++;
296    }
297
298    /**
299     * Returns the number of characters between the current position and the next instance of the input char
300     * @param c scan target
301     * @return offset between current position and next instance of target. -1 if not found.
302     */
303    int nextIndexOf(char c) {
304        // doesn't handle scanning for surrogates
305        bufferUp();
306        for (int i = bufPos; i < bufLength; i++) {
307            if (c == charBuf[i])
308                return i - bufPos;
309        }
310        return -1;
311    }
312
313    /**
314     * Returns the number of characters between the current position and the next instance of the input sequence
315     *
316     * @param seq scan target
317     * @return offset between current position and next instance of target. -1 if not found.
318     */
319    int nextIndexOf(CharSequence seq) {
320        bufferUp();
321        // doesn't handle scanning for surrogates
322        char startChar = seq.charAt(0);
323        for (int offset = bufPos; offset < bufLength; offset++) {
324            // scan to first instance of startchar:
325            if (startChar != charBuf[offset])
326                while(++offset < bufLength && startChar != charBuf[offset]) { /* empty */ }
327            int i = offset + 1;
328            int last = i + seq.length()-1;
329            if (offset < bufLength && last <= bufLength) {
330                for (int j = 1; i < last && seq.charAt(j) == charBuf[i]; i++, j++) { /* empty */ }
331                if (i == last) // found full sequence
332                    return offset - bufPos;
333            }
334        }
335        return -1;
336    }
337
338    /**
339     * Reads characters up to the specific char.
340     * @param c the delimiter
341     * @return the chars read
342     */
343    public String consumeTo(char c) {
344        int offset = nextIndexOf(c);
345        if (offset != -1) {
346            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
347            bufPos += offset;
348            return consumed;
349        } else {
350            return consumeToEnd();
351        }
352    }
353
354    /**
355     Reads the characters up to (but not including) the specified case-sensitive string.
356     <p>If the sequence is not found in the buffer, will return the remainder of the current buffered amount, less the
357     length of the sequence, such that this call may be repeated.
358     @param seq the delimiter
359     @return the chars read
360     */
361    public String consumeTo(String seq) {
362        int offset = nextIndexOf(seq);
363        if (offset != -1) {
364            String consumed = cacheString(charBuf, stringCache, bufPos, offset);
365            bufPos += offset;
366            return consumed;
367        } else if (bufLength - bufPos < seq.length()) {
368            // nextIndexOf() did a bufferUp(), so if the buffer is shorter than the search string, we must be at EOF
369            return consumeToEnd();
370        } else {
371            // the string we're looking for may be straddling a buffer boundary, so keep (length - 1) characters
372            // unread in case they contain the beginning of the search string
373            int endPos = bufLength - seq.length() + 1;
374            String consumed = cacheString(charBuf, stringCache, bufPos, endPos - bufPos);
375            bufPos = endPos;
376            return consumed;
377        }
378    }
379
380    /**
381     Read characters while the input predicate returns true.
382     @return characters read
383     */
384    String consumeMatching(CharPredicate func) {
385        return consumeMatching(func, -1);
386    }
387
388    /**
389     Read characters while the input predicate returns true, up to a maximum length.
390     @param func predicate to test
391     @param maxLength maximum length to read. -1 indicates no maximum
392     @return characters read
393     */
394    String consumeMatching(CharPredicate func, int maxLength) {
395        bufferUp();
396        int pos = bufPos;
397        final int start = pos;
398        final int remaining = bufLength;
399        final char[] val = charBuf;
400
401        while (pos < remaining && (maxLength == -1 || pos - start < maxLength) && func.test(val[pos])) {
402            pos++;
403        }
404
405        bufPos = pos;
406        return pos > start ? cacheString(charBuf, stringCache, start, pos -start) : "";
407    }
408
409    /**
410     Read characters until the first of any delimiters is found.
411     @param chars delimiters to scan for
412     @return characters read up to the matched delimiter.
413     */
414    public String consumeToAny(final char... chars) {
415        bufferUp();
416        int pos = bufPos;
417        final int start = pos;
418        final int remaining = bufLength;
419        final char[] val = charBuf;
420
421        scan:
422        while (pos < remaining) {
423            char c = val[pos];
424            for (char seek : chars)
425                if (c == seek) break scan;
426            pos++;
427        }
428
429        return consumeRange(start, pos);
430    }
431
432    /**
433     Read characters until either delimiter is found.
434     */
435    String consumeToAny(char c1, char c2) {
436        // monomorhpic to allow JIT to avoid virtual dispatch of e.g. consumeMatching(CharPredicate func)
437        bufferUp();
438        int pos = bufPos;
439        final int start = pos;
440        final int remaining = bufLength;
441        final char[] val = charBuf;
442
443        while (pos < remaining) {
444            char c = val[pos];
445            if (c == c1 || c == c2) break;
446            pos++;
447        }
448
449        return consumeRange(start, pos);
450    }
451
452    /**
453     Read characters until any delimiter is found.
454     */
455    String consumeToAny(char c1, char c2, char c3) {
456        bufferUp();
457        int pos = bufPos;
458        final int start = pos;
459        final int remaining = bufLength;
460        final char[] val = charBuf;
461
462        while (pos < remaining) {
463            char c = val[pos];
464            if (c == c1 || c == c2 || c == c3) break;
465            pos++;
466        }
467
468        return consumeRange(start, pos);
469    }
470
471    String consumeToAnySorted(final char... chars) {
472        bufferUp();
473        int pos = bufPos;
474        final int start = pos;
475        final int remaining = bufLength;
476        final char[] val = charBuf;
477
478        while (pos < remaining && Arrays.binarySearch(chars, val[pos]) < 0) {
479            pos++;
480        }
481
482        return consumeRange(start, pos);
483    }
484
485    /** Read HTML whitespace. */
486    String consumeWhitespace() {
487        bufferUp();
488        int start = bufPos;
489        int pos = start;
490        while (pos < bufLength && StringUtil.isWhitespace(charBuf[pos])) pos++;
491        return consumeRange(start, pos);
492    }
493
494    String consumeData() {
495        // consumes until &, <, null
496        return consumeToAny('&', '<', TokeniserState.nullChar);
497    }
498
499    String consumeAttributeQuoted(final boolean single) {
500        // null, " or ', &
501        char quote = single ? '\'' : '"';
502        return consumeToAny(TokeniserState.nullChar, '&', quote);
503    }
504
505    String consumeRawData() {
506        // <, null
507        return consumeToAny('<', TokeniserState.nullChar);
508    }
509
510    String consumeTagName() {
511        // '\t', '\n', '\r', '\f', ' ', '/', '>'
512        // NOTE: out of spec; does not stop and append on nullChar but eats
513        bufferUp();
514        int pos = bufPos;
515        final int start = pos;
516        final int remaining = bufLength;
517        final char[] val = charBuf;
518
519        while (pos < remaining) {
520            char c = val[pos];
521            switch (c) {
522                case '\t':
523                case '\n':
524                case '\r':
525                case '\f':
526                case ' ':
527                case '/':
528                case '>':
529                    return consumeRange(start, pos);
530            }
531            pos++;
532        }
533
534        return consumeRange(start, pos);
535    }
536
537    String consumeToEnd() {
538        bufferUp();
539        String data = cacheString(charBuf, stringCache, bufPos, bufLength - bufPos);
540        bufPos = bufLength;
541        return data;
542    }
543
544    /** Consumes ASCII letters used by the HTML tokenizer's name states. */
545    String consumeLetterSequence() {
546        return consumeMatching(StringUtil::isAsciiLetter);
547    }
548
549    String consumeLetterThenDigitSequence() {
550        bufferUp();
551        int start = bufPos;
552        while (bufPos < bufLength) {
553            if (StringUtil.isAsciiLetter(charBuf[bufPos])) bufPos++;
554            else break;
555        }
556        while (!isEmptyNoBufferUp()) {
557            if (StringUtil.isDigit(charBuf[bufPos])) bufPos++;
558            else break;
559        }
560
561        return cacheString(charBuf, stringCache, start, bufPos - start);
562    }
563
564    String consumeHexSequence() {
565        return consumeMatching(StringUtil::isHexDigit);
566    }
567
568    String consumeDigitSequence() {
569        return consumeMatching(c -> c >= '0' && c <= '9');
570    }
571
572    /**
573     Complete a scan by moving the reader and returning the matched range.
574     */
575    private String consumeRange(int start, int pos) {
576        bufPos = pos;
577        return pos > start ? cacheString(charBuf, stringCache, start, pos - start) : "";
578    }
579
580    boolean matches(char c) {
581        return !isEmpty() && charBuf[bufPos] == c;
582    }
583
584    boolean matches(String seq) {
585        bufferUp();
586        int scanLength = seq.length();
587        if (scanLength > bufLength - bufPos)
588            return false;
589
590        for (int offset = 0; offset < scanLength; offset++)
591            if (seq.charAt(offset) != charBuf[bufPos +offset])
592                return false;
593        return true;
594    }
595
596    /**
597     Checks if the current buffer position matches the sequence using ASCII case-insensitive matching.
598     */
599    boolean matchesIgnoreCase(String seq) {
600        bufferUp();
601        int scanLength = seq.length();
602        if (scanLength > bufLength - bufPos)
603            return false;
604
605        return rangeMatchesIgnoreCase(seq, bufPos);
606    }
607
608    private boolean rangeMatchesIgnoreCase(String seq, int start) {
609        for (int offset = 0; offset < seq.length(); offset++) {
610            char scan = seq.charAt(offset);
611            char target = charBuf[start + offset];
612            if (scan == target) continue;
613
614            scan = asciiLowerCase(scan);
615            target = asciiLowerCase(target);
616            if (scan != target) return false;
617        }
618        return true;
619    }
620
621    /**
622     Tests if the next character in the queue matches any of the characters in the sequence, case sensitively.
623     @param seq list of characters to check for
624     @return true if any matched, false if none did
625     */
626    boolean matchesAny(char... seq) {
627        if (isEmpty())
628            return false;
629
630        bufferUp();
631        char c = charBuf[bufPos];
632        for (char seek : seq) {
633            if (seek == c)
634                return true;
635        }
636        return false;
637    }
638
639    boolean matchesAnySorted(char[] seq) {
640        bufferUp();
641        return !isEmpty() && Arrays.binarySearch(seq, charBuf[bufPos]) >= 0;
642    }
643
644    /**
645     Checks if the current pos matches an ascii alpha (A-Z a-z) per https://infra.spec.whatwg.org/#ascii-alpha
646     @return if it matches or not
647     */
648    boolean matchesAsciiAlpha() {
649        if (isEmpty()) return false;
650        return StringUtil.isAsciiLetter(charBuf[bufPos]);
651    }
652
653    boolean matchesDigit() {
654        if (isEmpty()) return false;
655        return StringUtil.isDigit(charBuf[bufPos]);
656    }
657
658    boolean matchConsume(String seq) {
659        bufferUp();
660        if (matches(seq)) {
661            bufPos += seq.length();
662            return true;
663        } else {
664            return false;
665        }
666    }
667
668    boolean matchConsumeIgnoreCase(String seq) {
669        if (matchesIgnoreCase(seq)) {
670            bufPos += seq.length();
671            return true;
672        } else {
673            return false;
674        }
675    }
676
677    @Override
678    public String toString() {
679        if (bufLength - bufPos < 0) return "";
680        return new String(charBuf, bufPos, bufLength - bufPos);
681    }
682
683    /**
684     * Caches short strings, as a flyweight pattern, to reduce GC load. Just for this doc, to prevent leaks.
685     * <p />
686     * Simplistic, and on hash collisions just falls back to creating a new string, vs a full HashMap with Entry list.
687     * That saves both having to create objects as hash keys, and running through the entry list, at the expense of
688     * some more duplicates.
689     */
690    private static String cacheString(final char[] charBuf, final String[] stringCache, final int start, final int count) {
691        if (count > MaxStringCacheLen) // don't cache strings that are too big
692            return new String(charBuf, start, count);
693        if (count < 1)
694            return "";
695
696        // calculate hash:
697        int hash = 0;
698        int end = count + start;
699        for (int i = start; i < end; i++) {
700            hash = 31 * hash + charBuf[i];
701        }
702
703        // get from cache
704        final int index = hash & StringCacheSize - 1;
705        String cached = stringCache[index];
706
707        if (cached != null && rangeEquals(charBuf, start, count, cached)) // positive hit
708            return cached;
709        else {
710            cached = new String(charBuf, start, count);
711            stringCache[index] = cached; // add or replace, assuming most recently used are most likely to recur next
712        }
713
714        return cached;
715    }
716
717    /**
718     * Check if the value of the provided range equals the string.
719     */
720    static boolean rangeEquals(final char[] charBuf, final int start, int count, final String cached) {
721        if (count == cached.length()) {
722            int i = start;
723            int j = 0;
724            while (count-- != 0) {
725                if (charBuf[i++] != cached.charAt(j++))
726                    return false;
727            }
728            return true;
729        }
730        return false;
731    }
732
733    // just used for testing
734    boolean rangeEquals(final int start, final int count, final String cached) {
735        return rangeEquals(charBuf, start, count, cached);
736    }
737
738    @FunctionalInterface
739    interface CharPredicate {
740        boolean test(char c);
741    }
742}