001package org.jsoup.nodes;
002
003import org.jsoup.helper.DataUtil;
004import org.jsoup.internal.QuietAppendable;
005import org.jsoup.internal.StringUtil;
006import org.jsoup.helper.Validate;
007import org.jsoup.nodes.Document.OutputSettings;
008import org.jsoup.parser.CharacterReader;
009import org.jsoup.parser.Parser;
010
011import java.nio.CharBuffer;
012import java.nio.charset.Charset;
013import java.nio.charset.CharsetEncoder;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collections;
017import java.util.HashMap;
018
019import static org.jsoup.nodes.Entities.EscapeMode.base;
020import static org.jsoup.nodes.Entities.EscapeMode.extended;
021
022/**
023 * HTML entities, and escape routines. Source: <a href="http://www.w3.org/TR/html5/named-character-references.html#named-character-references">W3C
024 * HTML named character references</a>.
025 */
026public class Entities {
027    // constants for escape options:
028    static final int ForText = 0x1;
029    static final int ForAttribute = 0x2;
030    static final int Normalise = 0x4;
031    static final int TrimLeading = 0x8;
032    static final int TrimTrailing = 0x10;
033
034    private static final int empty = -1;
035    private static final String emptyName = "";
036    static final int codepointRadix = 36;
037    private static final char[] codeDelims = {',', ';'};
038    private static final HashMap<String, String> multipoints = new HashMap<>(); // name -> multiple character references
039
040    private static final int BaseCount = 106;
041    private static final ArrayList<String> baseSorted = new ArrayList<>(BaseCount); // names sorted longest first, for prefix matching
042
043    public enum EscapeMode {
044        /**
045         * Restricted entities suitable for XHTML output: lt, gt, amp, and quot only.
046         */
047        xhtml(EntitiesData.xmlPoints, 4),
048        /**
049         * Default HTML output entities.
050         */
051        base(EntitiesData.basePoints, 106),
052        /**
053         * Complete HTML entities.
054         */
055        extended(EntitiesData.fullPoints, 2125);
056
057        static {
058            // sort the base names by length, for prefix matching
059            Collections.addAll(baseSorted, base.nameKeys);
060            baseSorted.sort((a, b) -> b.length() - a.length());
061        }
062
063        // table of named references to their codepoints. sorted so we can binary search. built by BuildEntities.
064        private String[] nameKeys;
065        private int[] codeVals; // limitation is the few references with multiple characters; those go into multipoints.
066
067        // table of codepoints to named entities.
068        private int[] codeKeys; // we don't support multicodepoints to single named value currently
069        private String[] nameVals;
070
071        EscapeMode(String file, int size) {
072            load(this, file, size);
073        }
074
075        int codepointForName(final String name) {
076            int index = Arrays.binarySearch(nameKeys, name);
077            return index >= 0 ? codeVals[index] : empty;
078        }
079
080        String nameForCodepoint(final int codepoint) {
081            final int index = Arrays.binarySearch(codeKeys, codepoint);
082            if (index >= 0) {
083                // the results are ordered so lower case versions of same codepoint come after uppercase, and we prefer to emit lower
084                // (and binary search for same item with multi results is undefined
085                return (index < nameVals.length - 1 && codeKeys[index + 1] == codepoint) ?
086                    nameVals[index + 1] : nameVals[index];
087            }
088            return emptyName;
089        }
090    }
091
092    private Entities() {
093    }
094
095    /**
096     * Check if the input is a known named entity
097     *
098     * @param name the possible entity name (e.g. "lt" or "amp")
099     * @return true if a known named entity
100     */
101    public static boolean isNamedEntity(final String name) {
102        return extended.codepointForName(name) != empty;
103    }
104
105    /**
106     * Check if the input is a known named entity in the base entity set.
107     *
108     * @param name the possible entity name (e.g. "lt" or "amp")
109     * @return true if a known named entity in the base set
110     * @see #isNamedEntity(String)
111     */
112    public static boolean isBaseNamedEntity(final String name) {
113        return base.codepointForName(name) != empty;
114    }
115
116    /**
117     * Get the character(s) represented by the named entity
118     *
119     * @param name entity (e.g. "lt" or "amp")
120     * @return the string value of the character(s) represented by this entity, or "" if not defined
121     */
122    public static String getByName(String name) {
123        String val = multipoints.get(name);
124        if (val != null)
125            return val;
126        int codepoint = extended.codepointForName(name);
127        if (codepoint != empty)
128            return new String(new int[]{codepoint}, 0, 1);
129        return emptyName;
130    }
131
132    public static int codepointsForName(final String name, final int[] codepoints) {
133        String val = multipoints.get(name);
134        if (val != null) {
135            codepoints[0] = val.codePointAt(0);
136            codepoints[1] = val.codePointAt(1);
137            return 2;
138        }
139        int codepoint = extended.codepointForName(name);
140        if (codepoint != empty) {
141            codepoints[0] = codepoint;
142            return 1;
143        }
144        return 0;
145    }
146
147    /**
148     Finds the longest base named entity that is a prefix of the input. That is, input "notit" would return "not".
149
150     @return longest entity name that is a prefix of the input, or "" if no entity matches
151     */
152    public static String findPrefix(String input) {
153        for (String name : baseSorted) {
154            if (input.startsWith(name)) return name;
155        }
156        return emptyName;
157        // if perf critical, could look at using a Trie vs a scan
158    }
159
160    /**
161     HTML escape an input string. That is, {@code <} is returned as {@code &lt;}. The escaped string is suitable for use
162     both in attributes and in text data.
163     @param data the un-escaped string to escape
164     @param out the output settings to use. This configures the character set escaped against (that is, if a
165     character is supported in the output character set, it doesn't have to be escaped), and also HTML or XML
166     settings.
167     @return the escaped string
168     */
169    public static String escape(String data, OutputSettings out) {
170        return escapeString(data, out.escapeMode(), out.charset());
171    }
172
173    /**
174     HTML escape an input string, using the default settings (UTF-8, base entities). That is, {@code <} is
175     returned as {@code &lt;}. The escaped string is suitable for use both in attributes and in text data.
176     @param data the un-escaped string to escape
177     @return the escaped string
178     @see #escape(String, OutputSettings)
179     */
180    public static String escape(String data) {
181        return escapeString(data, base, DataUtil.UTF_8);
182    }
183
184    private static String escapeString(String data, EscapeMode escapeMode, Charset charset) {
185        if (data == null) return "";
186        StringBuilder sb = StringUtil.borrowBuilder();
187        doEscape(data, QuietAppendable.wrap(sb), escapeMode, charset, ForText | ForAttribute);
188        return StringUtil.releaseBuilder(sb);
189    }
190
191    static void escape(QuietAppendable accum, String data, OutputSettings out, int options) {
192        doEscape(data, accum, out.escapeMode(), out.charset(), options);
193    }
194
195    private static void doEscape(String data, QuietAppendable accum, EscapeMode mode, Charset charset, int options) {
196        final CoreCharset coreCharset = CoreCharset.byName(charset.name());
197        final CharsetEncoder fallback = encoderFor(charset);
198        final int length = data.length();
199
200        int codePoint;
201        boolean lastWasWhite = false;
202        boolean reachedNonWhite = false;
203        boolean skipped = false;
204        for (int offset = 0; offset < length; offset += Character.charCount(codePoint)) {
205            codePoint = data.codePointAt(offset);
206
207            if ((options & Normalise) != 0) {
208                if (StringUtil.isWhitespace(codePoint)) {
209                    if ((options & TrimLeading) != 0 && !reachedNonWhite) continue;
210                    if (lastWasWhite) continue;
211                    if ((options & TrimTrailing) != 0) {
212                        skipped = true;
213                        continue;
214                    }
215                    accum.append(' ');
216                    lastWasWhite = true;
217                    continue;
218                } else {
219                    lastWasWhite = false;
220                    reachedNonWhite = true;
221                    if (skipped) {
222                        accum.append(' '); // wasn't the end, so need to place a normalized space
223                        skipped = false;
224                    }
225                }
226            }
227            appendEscaped(codePoint, accum, options, mode, coreCharset, fallback);
228        }
229    }
230
231    private static void appendEscaped(int codePoint, QuietAppendable accum, int options, EscapeMode escapeMode,
232        CoreCharset coreCharset, CharsetEncoder fallback) {
233        // specific character range for xml 1.0; drop (not encode) if so
234        if (EscapeMode.xhtml == escapeMode && !isValidXmlChar(codePoint)) {
235            return;
236        }
237
238        // surrogate pairs, split implementation for efficiency on single char common case (saves creating strings, char[]):
239        final char c = (char) codePoint;
240        if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) {
241            // html specific and required escapes:
242            switch (c) {
243                case '&':
244                    accum.append("&amp;");
245                    break;
246                case 0xA0:
247                    appendNbsp(accum, escapeMode);
248                    break;
249                case '<':
250                    accum.append("&lt;");
251                    break;
252                case '>':
253                    accum.append("&gt;");
254                    break;
255                case '"':
256                    if ((options & ForAttribute) != 0) accum.append("&quot;");
257                    else accum.append(c);
258                    break;
259                case '\'':
260                    // special case for the Entities.escape(string) method when we are maximally escaping. Otherwise, because we output attributes in "", there's no need to escape.
261                    appendApos(accum, options, escapeMode);
262                    break;
263                // we escape ascii control <x20 (other than tab, line-feed, carriage return) for XML compliance (required) and HTML ease of reading (not required) - https://www.w3.org/TR/xml/#charsets
264                case 0x9:
265                case 0xA:
266                case 0xD:
267                    accum.append(c);
268                    break;
269                default:
270                    if (c < 0x20 || !canEncode(coreCharset, codePoint, fallback)) appendEncoded(accum, escapeMode, codePoint);
271                    else accum.append(c);
272            }
273        } else {
274            if (canEncode(coreCharset, codePoint, fallback)) {
275                // reads into charBuf - we go through these steps to avoid GC objects as much as possible (would be a new String and a new char[2] for each character)
276                char[] chars = charBuf.get();
277                int len = Character.toChars(codePoint, chars, 0);
278                accum.append(chars, 0, len);
279            } else {
280                appendEncoded(accum, escapeMode, codePoint);
281            }
282        }
283    }
284
285    private static final ThreadLocal<char[]> charBuf = ThreadLocal.withInitial(() -> new char[2]);
286
287    private static void appendNbsp(QuietAppendable accum, EscapeMode escapeMode) {
288        if (escapeMode != EscapeMode.xhtml) accum.append("&nbsp;");
289        else accum.append("&#xa0;");
290    }
291
292    private static void appendApos(QuietAppendable accum, int options, EscapeMode escapeMode) {
293        if ((options & ForAttribute) != 0 && (options & ForText) != 0) {
294            if (escapeMode == EscapeMode.xhtml) accum.append("&#x27;");
295            else accum.append("&apos;");
296        } else {
297            accum.append('\'');
298        }
299    }
300
301    private static void appendEncoded(QuietAppendable accum, EscapeMode escapeMode, int codePoint) {
302        final String name = escapeMode.nameForCodepoint(codePoint);
303        if (!emptyName.equals(name)) // ok for identity check
304            accum.append('&').append(name).append(';');
305        else
306            accum.append("&#x").append(Integer.toHexString(codePoint)).append(';');
307    }
308
309    /**
310     * Un-escape an HTML escaped string. That is, {@code &lt;} is returned as {@code <}.
311     *
312     * @param string the HTML string to un-escape
313     * @return the unescaped string
314     */
315    public static String unescape(String string) {
316        return unescape(string, false);
317    }
318
319    /**
320     * Unescape the input string.
321     *
322     * @param string to un-HTML-escape
323     * @param strict if "strict" (that is, requires trailing ';' char, otherwise that's optional)
324     * @return unescaped string
325     */
326    static String unescape(String string, boolean strict) {
327        return Parser.unescapeEntities(string, strict);
328    }
329
330    /*
331     * Provides a fast-path for Encoder.canEncode, which drastically improves performance on Android post JellyBean.
332     * After KitKat, the implementation of canEncode degrades to the point of being useless. For non ASCII or UTF,
333     * performance may be bad. We can add more encoders for common character sets that are impacted by performance
334     * issues on Android if required.
335     *
336     * Benchmarks:     *
337     * OLD toHtml() impl v New (fastpath) in millis
338     * Wiki: 1895, 16
339     * CNN: 6378, 55
340     * Alterslash: 3013, 28
341     * Jsoup: 167, 2
342     */
343    private static boolean canEncode(final CoreCharset charset, final int codePoint, final CharsetEncoder fallback) {
344        // todo add more charset tests if impacted by Android's bad perf in canEncode
345        switch (charset) {
346            case ascii:
347                return codePoint < 0x80;
348            case utf:
349                // reject unpaired UTF-16 surrogate code units; valid supplementary code points are outside this range
350                return codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE;
351            default:
352                if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT)
353                    return fallback.canEncode((char) codePoint);
354
355                // check the complete UTF-16 pair; checking only the low 16 bits could accept an unencodable code point
356                char[] chars = charBuf.get();
357                int len = Character.toChars(codePoint, chars, 0);
358                return fallback.canEncode(CharBuffer.wrap(chars, 0, len));
359        }
360    }
361
362    private static boolean isValidXmlChar(int codePoint) {
363        // https://www.w3.org/TR/2006/REC-xml-20060816/Overview.html#charsets
364        // Char    ::=          #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]  any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
365        return (codePoint == 0x9 || codePoint == 0xA || codePoint == 0xD || (codePoint >= 0x20 && codePoint <= 0xD7FF)
366            || (codePoint >= 0xE000 && codePoint <= 0xFFFD) || (codePoint >= 0x10000 && codePoint <= 0x10FFFF));
367    }
368
369    enum CoreCharset {
370        ascii, utf, fallback;
371
372        static CoreCharset byName(final String name) {
373            if (name.equals("US-ASCII"))
374                return ascii;
375            if (name.startsWith("UTF-")) // covers UTF-8, UTF-16, et al
376                return utf;
377            return fallback;
378        }
379    }
380
381    // cache the last used fallback encoder to save recreating on every use
382    private static final ThreadLocal<CharsetEncoder> LocalEncoder = new ThreadLocal<>();
383    private static CharsetEncoder encoderFor(Charset charset) {
384        CharsetEncoder encoder = LocalEncoder.get();
385        if (encoder == null || !encoder.charset().equals(charset)) {
386            encoder = charset.newEncoder();
387            LocalEncoder.set(encoder);
388        }
389        return encoder;
390    }
391
392    private static void load(EscapeMode e, String pointsData, int size) {
393        e.nameKeys = new String[size];
394        e.codeVals = new int[size];
395        e.codeKeys = new int[size];
396        e.nameVals = new String[size];
397
398        int i = 0;
399        try (CharacterReader reader = new CharacterReader(pointsData)) {
400            while (!reader.isEmpty()) {
401                // NotNestedLessLess=10913,824;1887&
402
403                final String name = reader.consumeTo('=');
404                reader.advance();
405                final int cp1 = Integer.parseInt(reader.consumeToAny(codeDelims), codepointRadix);
406                final char codeDelim = reader.current();
407                reader.advance();
408                final int cp2;
409                if (codeDelim == ',') {
410                    cp2 = Integer.parseInt(reader.consumeTo(';'), codepointRadix);
411                    reader.advance();
412                } else {
413                    cp2 = empty;
414                }
415                final String indexS = reader.consumeTo('&');
416                final int index = Integer.parseInt(indexS, codepointRadix);
417                reader.advance();
418
419                e.nameKeys[i] = name;
420                e.codeVals[i] = cp1;
421                e.codeKeys[index] = cp1;
422                e.nameVals[index] = name;
423
424                if (cp2 != empty) {
425                    multipoints.put(name, new String(new int[]{cp1, cp2}, 0, 2));
426                }
427                i++;
428            }
429
430            Validate.isTrue(i == size, "Unexpected count of entities loaded");
431        }
432    }
433}