001package org.jsoup.nodes; 002 003import org.jsoup.helper.DataUtil; 004import org.jsoup.internal.QuietAppendable; 005import org.jsoup.internal.StringUtil; 006import org.jsoup.helper.Validate; 007import org.jsoup.nodes.Document.OutputSettings; 008import org.jsoup.parser.CharacterReader; 009import org.jsoup.parser.Parser; 010 011import java.nio.CharBuffer; 012import java.nio.charset.Charset; 013import java.nio.charset.CharsetEncoder; 014import java.util.ArrayList; 015import java.util.Arrays; 016import java.util.Collections; 017import java.util.HashMap; 018 019import static org.jsoup.nodes.Entities.EscapeMode.base; 020import static org.jsoup.nodes.Entities.EscapeMode.extended; 021 022/** 023 * HTML entities, and escape routines. Source: <a href="http://www.w3.org/TR/html5/named-character-references.html#named-character-references">W3C 024 * HTML named character references</a>. 025 */ 026public class Entities { 027 // constants for escape options: 028 static final int ForText = 0x1; 029 static final int ForAttribute = 0x2; 030 static final int Normalise = 0x4; 031 static final int TrimLeading = 0x8; 032 static final int TrimTrailing = 0x10; 033 034 private static final int empty = -1; 035 private static final String emptyName = ""; 036 static final int codepointRadix = 36; 037 private static final char[] codeDelims = {',', ';'}; 038 private static final HashMap<String, String> multipoints = new HashMap<>(); // name -> multiple character references 039 040 private static final int BaseCount = 106; 041 private static final ArrayList<String> baseSorted = new ArrayList<>(BaseCount); // names sorted longest first, for prefix matching 042 043 public enum EscapeMode { 044 /** 045 * Restricted entities suitable for XHTML output: lt, gt, amp, and quot only. 046 */ 047 xhtml(EntitiesData.xmlPoints, 4), 048 /** 049 * Default HTML output entities. 050 */ 051 base(EntitiesData.basePoints, 106), 052 /** 053 * Complete HTML entities. 054 */ 055 extended(EntitiesData.fullPoints, 2125); 056 057 static { 058 // sort the base names by length, for prefix matching 059 Collections.addAll(baseSorted, base.nameKeys); 060 baseSorted.sort((a, b) -> b.length() - a.length()); 061 } 062 063 // table of named references to their codepoints. sorted so we can binary search. built by BuildEntities. 064 private String[] nameKeys; 065 private int[] codeVals; // limitation is the few references with multiple characters; those go into multipoints. 066 067 // table of codepoints to named entities. 068 private int[] codeKeys; // we don't support multicodepoints to single named value currently 069 private String[] nameVals; 070 071 EscapeMode(String file, int size) { 072 load(this, file, size); 073 } 074 075 int codepointForName(final String name) { 076 int index = Arrays.binarySearch(nameKeys, name); 077 return index >= 0 ? codeVals[index] : empty; 078 } 079 080 String nameForCodepoint(final int codepoint) { 081 final int index = Arrays.binarySearch(codeKeys, codepoint); 082 if (index >= 0) { 083 // the results are ordered so lower case versions of same codepoint come after uppercase, and we prefer to emit lower 084 // (and binary search for same item with multi results is undefined 085 return (index < nameVals.length - 1 && codeKeys[index + 1] == codepoint) ? 086 nameVals[index + 1] : nameVals[index]; 087 } 088 return emptyName; 089 } 090 } 091 092 private Entities() { 093 } 094 095 /** 096 * Check if the input is a known named entity 097 * 098 * @param name the possible entity name (e.g. "lt" or "amp") 099 * @return true if a known named entity 100 */ 101 public static boolean isNamedEntity(final String name) { 102 return extended.codepointForName(name) != empty; 103 } 104 105 /** 106 * Check if the input is a known named entity in the base entity set. 107 * 108 * @param name the possible entity name (e.g. "lt" or "amp") 109 * @return true if a known named entity in the base set 110 * @see #isNamedEntity(String) 111 */ 112 public static boolean isBaseNamedEntity(final String name) { 113 return base.codepointForName(name) != empty; 114 } 115 116 /** 117 * Get the character(s) represented by the named entity 118 * 119 * @param name entity (e.g. "lt" or "amp") 120 * @return the string value of the character(s) represented by this entity, or "" if not defined 121 */ 122 public static String getByName(String name) { 123 String val = multipoints.get(name); 124 if (val != null) 125 return val; 126 int codepoint = extended.codepointForName(name); 127 if (codepoint != empty) 128 return new String(new int[]{codepoint}, 0, 1); 129 return emptyName; 130 } 131 132 public static int codepointsForName(final String name, final int[] codepoints) { 133 String val = multipoints.get(name); 134 if (val != null) { 135 codepoints[0] = val.codePointAt(0); 136 codepoints[1] = val.codePointAt(1); 137 return 2; 138 } 139 int codepoint = extended.codepointForName(name); 140 if (codepoint != empty) { 141 codepoints[0] = codepoint; 142 return 1; 143 } 144 return 0; 145 } 146 147 /** 148 Finds the longest base named entity that is a prefix of the input. That is, input "notit" would return "not". 149 150 @return longest entity name that is a prefix of the input, or "" if no entity matches 151 */ 152 public static String findPrefix(String input) { 153 for (String name : baseSorted) { 154 if (input.startsWith(name)) return name; 155 } 156 return emptyName; 157 // if perf critical, could look at using a Trie vs a scan 158 } 159 160 /** 161 HTML escape an input string. That is, {@code <} is returned as {@code <}. The escaped string is suitable for use 162 both in attributes and in text data. 163 @param data the un-escaped string to escape 164 @param out the output settings to use. This configures the character set escaped against (that is, if a 165 character is supported in the output character set, it doesn't have to be escaped), and also HTML or XML 166 settings. 167 @return the escaped string 168 */ 169 public static String escape(String data, OutputSettings out) { 170 return escapeString(data, out.escapeMode(), out.charset()); 171 } 172 173 /** 174 HTML escape an input string, using the default settings (UTF-8, base entities). That is, {@code <} is 175 returned as {@code <}. The escaped string is suitable for use both in attributes and in text data. 176 @param data the un-escaped string to escape 177 @return the escaped string 178 @see #escape(String, OutputSettings) 179 */ 180 public static String escape(String data) { 181 return escapeString(data, base, DataUtil.UTF_8); 182 } 183 184 private static String escapeString(String data, EscapeMode escapeMode, Charset charset) { 185 if (data == null) return ""; 186 StringBuilder sb = StringUtil.borrowBuilder(); 187 doEscape(data, QuietAppendable.wrap(sb), escapeMode, charset, ForText | ForAttribute); 188 return StringUtil.releaseBuilder(sb); 189 } 190 191 static void escape(QuietAppendable accum, String data, OutputSettings out, int options) { 192 doEscape(data, accum, out.escapeMode(), out.charset(), options); 193 } 194 195 private static void doEscape(String data, QuietAppendable accum, EscapeMode mode, Charset charset, int options) { 196 final CoreCharset coreCharset = CoreCharset.byName(charset.name()); 197 final CharsetEncoder fallback = encoderFor(charset); 198 final int length = data.length(); 199 200 int codePoint; 201 boolean lastWasWhite = false; 202 boolean reachedNonWhite = false; 203 boolean skipped = false; 204 for (int offset = 0; offset < length; offset += Character.charCount(codePoint)) { 205 codePoint = data.codePointAt(offset); 206 207 if ((options & Normalise) != 0) { 208 if (StringUtil.isWhitespace(codePoint)) { 209 if ((options & TrimLeading) != 0 && !reachedNonWhite) continue; 210 if (lastWasWhite) continue; 211 if ((options & TrimTrailing) != 0) { 212 skipped = true; 213 continue; 214 } 215 accum.append(' '); 216 lastWasWhite = true; 217 continue; 218 } else { 219 lastWasWhite = false; 220 reachedNonWhite = true; 221 if (skipped) { 222 accum.append(' '); // wasn't the end, so need to place a normalized space 223 skipped = false; 224 } 225 } 226 } 227 appendEscaped(codePoint, accum, options, mode, coreCharset, fallback); 228 } 229 } 230 231 private static void appendEscaped(int codePoint, QuietAppendable accum, int options, EscapeMode escapeMode, 232 CoreCharset coreCharset, CharsetEncoder fallback) { 233 // specific character range for xml 1.0; drop (not encode) if so 234 if (EscapeMode.xhtml == escapeMode && !isValidXmlChar(codePoint)) { 235 return; 236 } 237 238 // surrogate pairs, split implementation for efficiency on single char common case (saves creating strings, char[]): 239 final char c = (char) codePoint; 240 if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) { 241 // html specific and required escapes: 242 switch (c) { 243 case '&': 244 accum.append("&"); 245 break; 246 case 0xA0: 247 appendNbsp(accum, escapeMode); 248 break; 249 case '<': 250 accum.append("<"); 251 break; 252 case '>': 253 accum.append(">"); 254 break; 255 case '"': 256 if ((options & ForAttribute) != 0) accum.append("""); 257 else accum.append(c); 258 break; 259 case '\'': 260 // special case for the Entities.escape(string) method when we are maximally escaping. Otherwise, because we output attributes in "", there's no need to escape. 261 appendApos(accum, options, escapeMode); 262 break; 263 // we escape ascii control <x20 (other than tab, line-feed, carriage return) for XML compliance (required) and HTML ease of reading (not required) - https://www.w3.org/TR/xml/#charsets 264 case 0x9: 265 case 0xA: 266 case 0xD: 267 accum.append(c); 268 break; 269 default: 270 if (c < 0x20 || !canEncode(coreCharset, codePoint, fallback)) appendEncoded(accum, escapeMode, codePoint); 271 else accum.append(c); 272 } 273 } else { 274 if (canEncode(coreCharset, codePoint, fallback)) { 275 // reads into charBuf - we go through these steps to avoid GC objects as much as possible (would be a new String and a new char[2] for each character) 276 char[] chars = charBuf.get(); 277 int len = Character.toChars(codePoint, chars, 0); 278 accum.append(chars, 0, len); 279 } else { 280 appendEncoded(accum, escapeMode, codePoint); 281 } 282 } 283 } 284 285 private static final ThreadLocal<char[]> charBuf = ThreadLocal.withInitial(() -> new char[2]); 286 287 private static void appendNbsp(QuietAppendable accum, EscapeMode escapeMode) { 288 if (escapeMode != EscapeMode.xhtml) accum.append(" "); 289 else accum.append(" "); 290 } 291 292 private static void appendApos(QuietAppendable accum, int options, EscapeMode escapeMode) { 293 if ((options & ForAttribute) != 0 && (options & ForText) != 0) { 294 if (escapeMode == EscapeMode.xhtml) accum.append("'"); 295 else accum.append("'"); 296 } else { 297 accum.append('\''); 298 } 299 } 300 301 private static void appendEncoded(QuietAppendable accum, EscapeMode escapeMode, int codePoint) { 302 final String name = escapeMode.nameForCodepoint(codePoint); 303 if (!emptyName.equals(name)) // ok for identity check 304 accum.append('&').append(name).append(';'); 305 else 306 accum.append("&#x").append(Integer.toHexString(codePoint)).append(';'); 307 } 308 309 /** 310 * Un-escape an HTML escaped string. That is, {@code <} is returned as {@code <}. 311 * 312 * @param string the HTML string to un-escape 313 * @return the unescaped string 314 */ 315 public static String unescape(String string) { 316 return unescape(string, false); 317 } 318 319 /** 320 * Unescape the input string. 321 * 322 * @param string to un-HTML-escape 323 * @param strict if "strict" (that is, requires trailing ';' char, otherwise that's optional) 324 * @return unescaped string 325 */ 326 static String unescape(String string, boolean strict) { 327 return Parser.unescapeEntities(string, strict); 328 } 329 330 /* 331 * Provides a fast-path for Encoder.canEncode, which drastically improves performance on Android post JellyBean. 332 * After KitKat, the implementation of canEncode degrades to the point of being useless. For non ASCII or UTF, 333 * performance may be bad. We can add more encoders for common character sets that are impacted by performance 334 * issues on Android if required. 335 * 336 * Benchmarks: * 337 * OLD toHtml() impl v New (fastpath) in millis 338 * Wiki: 1895, 16 339 * CNN: 6378, 55 340 * Alterslash: 3013, 28 341 * Jsoup: 167, 2 342 */ 343 private static boolean canEncode(final CoreCharset charset, final int codePoint, final CharsetEncoder fallback) { 344 // todo add more charset tests if impacted by Android's bad perf in canEncode 345 switch (charset) { 346 case ascii: 347 return codePoint < 0x80; 348 case utf: 349 // reject unpaired UTF-16 surrogate code units; valid supplementary code points are outside this range 350 return codePoint < Character.MIN_SURROGATE || codePoint > Character.MAX_SURROGATE; 351 default: 352 if (codePoint < Character.MIN_SUPPLEMENTARY_CODE_POINT) 353 return fallback.canEncode((char) codePoint); 354 355 // check the complete UTF-16 pair; checking only the low 16 bits could accept an unencodable code point 356 char[] chars = charBuf.get(); 357 int len = Character.toChars(codePoint, chars, 0); 358 return fallback.canEncode(CharBuffer.wrap(chars, 0, len)); 359 } 360 } 361 362 private static boolean isValidXmlChar(int codePoint) { 363 // https://www.w3.org/TR/2006/REC-xml-20060816/Overview.html#charsets 364 // Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. 365 return (codePoint == 0x9 || codePoint == 0xA || codePoint == 0xD || (codePoint >= 0x20 && codePoint <= 0xD7FF) 366 || (codePoint >= 0xE000 && codePoint <= 0xFFFD) || (codePoint >= 0x10000 && codePoint <= 0x10FFFF)); 367 } 368 369 enum CoreCharset { 370 ascii, utf, fallback; 371 372 static CoreCharset byName(final String name) { 373 if (name.equals("US-ASCII")) 374 return ascii; 375 if (name.startsWith("UTF-")) // covers UTF-8, UTF-16, et al 376 return utf; 377 return fallback; 378 } 379 } 380 381 // cache the last used fallback encoder to save recreating on every use 382 private static final ThreadLocal<CharsetEncoder> LocalEncoder = new ThreadLocal<>(); 383 private static CharsetEncoder encoderFor(Charset charset) { 384 CharsetEncoder encoder = LocalEncoder.get(); 385 if (encoder == null || !encoder.charset().equals(charset)) { 386 encoder = charset.newEncoder(); 387 LocalEncoder.set(encoder); 388 } 389 return encoder; 390 } 391 392 private static void load(EscapeMode e, String pointsData, int size) { 393 e.nameKeys = new String[size]; 394 e.codeVals = new int[size]; 395 e.codeKeys = new int[size]; 396 e.nameVals = new String[size]; 397 398 int i = 0; 399 try (CharacterReader reader = new CharacterReader(pointsData)) { 400 while (!reader.isEmpty()) { 401 // NotNestedLessLess=10913,824;1887& 402 403 final String name = reader.consumeTo('='); 404 reader.advance(); 405 final int cp1 = Integer.parseInt(reader.consumeToAny(codeDelims), codepointRadix); 406 final char codeDelim = reader.current(); 407 reader.advance(); 408 final int cp2; 409 if (codeDelim == ',') { 410 cp2 = Integer.parseInt(reader.consumeTo(';'), codepointRadix); 411 reader.advance(); 412 } else { 413 cp2 = empty; 414 } 415 final String indexS = reader.consumeTo('&'); 416 final int index = Integer.parseInt(indexS, codepointRadix); 417 reader.advance(); 418 419 e.nameKeys[i] = name; 420 e.codeVals[i] = cp1; 421 e.codeKeys[index] = cp1; 422 e.nameVals[index] = name; 423 424 if (cp2 != empty) { 425 multipoints.put(name, new String(new int[]{cp1, cp2}, 0, 2)); 426 } 427 i++; 428 } 429 430 Validate.isTrue(i == size, "Unexpected count of entities loaded"); 431 } 432 } 433}