001package org.jsoup.helper;
002
003import org.jsoup.internal.Normalizer;
004import org.jsoup.internal.StringUtil;
005import org.jsoup.nodes.Attribute;
006import org.jsoup.parser.HtmlTreeBuilder;
007import org.jsoup.parser.Parser;
008import org.jsoup.select.NodeVisitor;
009import org.jsoup.select.Selector;
010import org.w3c.dom.Comment;
011import org.w3c.dom.DOMException;
012import org.w3c.dom.DOMImplementation;
013import org.w3c.dom.Document;
014import org.w3c.dom.DocumentType;
015import org.w3c.dom.Element;
016import org.w3c.dom.Node;
017import org.w3c.dom.NodeList;
018import org.w3c.dom.Text;
019import org.jspecify.annotations.Nullable;
020
021import javax.xml.parsers.DocumentBuilder;
022import javax.xml.parsers.DocumentBuilderFactory;
023import javax.xml.parsers.ParserConfigurationException;
024import javax.xml.transform.OutputKeys;
025import javax.xml.transform.Transformer;
026import javax.xml.transform.TransformerException;
027import javax.xml.transform.TransformerFactory;
028import javax.xml.transform.dom.DOMSource;
029import javax.xml.transform.stream.StreamResult;
030import javax.xml.xpath.XPathConstants;
031import javax.xml.xpath.XPathExpression;
032import javax.xml.xpath.XPathExpressionException;
033import javax.xml.xpath.XPathFactory;
034import javax.xml.xpath.XPathFactoryConfigurationException;
035import java.io.StringWriter;
036import java.util.ArrayList;
037import java.util.HashMap;
038import java.util.List;
039import java.util.Map;
040import java.util.Properties;
041
042import static javax.xml.transform.OutputKeys.METHOD;
043import static org.jsoup.nodes.Document.OutputSettings.Syntax;
044
045/**
046 * Helper class to transform a {@link org.jsoup.nodes.Document} to a {@link org.w3c.dom.Document org.w3c.dom.Document},
047 * for integration with toolsets that use the W3C DOM.
048 */
049public class W3CDom {
050    /** For W3C Documents created by this class, this property is set on each node to link back to the original jsoup node. */
051    public static final String SourceProperty = "jsoupSource";
052    private static final String ContextProperty = "jsoupContextSource"; // tracks the jsoup context element on w3c doc
053    private static final String ContextNodeProperty = "jsoupContextNode"; // the w3c node used as the creating context
054
055    /**
056     To get support for XPath versions > 1, set this property to the classname of an alternate XPathFactory
057     implementation. (For e.g. {@code net.sf.saxon.xpath.XPathFactoryImpl}).
058     */
059    public static final String XPathFactoryProperty = "javax.xml.xpath.XPathFactory:jsoup";
060
061    protected DocumentBuilderFactory factory;
062    private boolean namespaceAware = true; // false when using selectXpath, for user's query convenience
063
064    public W3CDom() {
065        factory = DocumentBuilderFactory.newInstance();
066        factory.setNamespaceAware(true);
067    }
068
069    /**
070     Returns if this W3C DOM is namespace aware. By default, this will be {@code true}, but is disabled for simplicity
071     when using XPath selectors in {@link org.jsoup.nodes.Element#selectXpath(String)}.
072     @return the current namespace aware setting.
073     */
074    public boolean namespaceAware() {
075        return namespaceAware;
076    }
077
078    /**
079     Update the namespace aware setting. This impacts the factory that is used to create W3C nodes from jsoup nodes.
080     <p>For HTML documents, controls if the document will be in the default {@code http://www.w3.org/1999/xhtml}
081     namespace if otherwise unset.</p>.
082     @param namespaceAware the updated setting
083     @return this W3CDom, for chaining.
084     */
085    public W3CDom namespaceAware(boolean namespaceAware) {
086        this.namespaceAware = namespaceAware;
087        factory.setNamespaceAware(namespaceAware);
088        return this;
089    }
090
091    /**
092     * Converts a jsoup DOM to a W3C DOM.
093     *
094     * @param in jsoup Document
095     * @return W3C Document
096     */
097    public static Document convert(org.jsoup.nodes.Document in) {
098        return (new W3CDom().fromJsoup(in));
099    }
100
101    /**
102     * Serialize a W3C document to a String. Provide Properties to define output settings including if HTML or XML. If
103     * you don't provide the properties ({@code null}), the output will be auto-detected based on the content of the
104     * document.
105     *
106     * @param doc Document
107     * @param properties (optional/nullable) the output properties to use. See {@link
108     *     Transformer#setOutputProperties(Properties)} and {@link OutputKeys}
109     * @return Document as string
110     * @see #OutputHtml
111     * @see #OutputXml
112     * @see OutputKeys#ENCODING
113     * @see OutputKeys#OMIT_XML_DECLARATION
114     * @see OutputKeys#STANDALONE
115     * @see OutputKeys#DOCTYPE_PUBLIC
116     * @see OutputKeys#CDATA_SECTION_ELEMENTS
117     * @see OutputKeys#INDENT
118     * @see OutputKeys#MEDIA_TYPE
119     */
120    public static String asString(Document doc, @Nullable Map<String, String> properties) {
121        try {
122            DOMSource domSource = new DOMSource(doc);
123            StringWriter writer = new StringWriter();
124            StreamResult result = new StreamResult(writer);
125            TransformerFactory tf = TransformerFactory.newInstance();
126            Transformer transformer = tf.newTransformer();
127            if (properties != null)
128                transformer.setOutputProperties(propertiesFromMap(properties));
129
130            if (doc.getDoctype() != null) {
131                DocumentType doctype = doc.getDoctype();
132                if (!StringUtil.isBlank(doctype.getPublicId()))
133                    transformer.setOutputProperty(OutputKeys.DOCTYPE_PUBLIC, doctype.getPublicId());
134                if (!StringUtil.isBlank(doctype.getSystemId()))
135                    transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, doctype.getSystemId());
136                    // handle <!doctype html> for legacy dom.
137                else if (doctype.getName().equalsIgnoreCase("html")
138                    && StringUtil.isBlank(doctype.getPublicId())
139                    && StringUtil.isBlank(doctype.getSystemId()))
140                    transformer.setOutputProperty(OutputKeys.DOCTYPE_SYSTEM, "about:legacy-compat");
141            }
142
143            transformer.transform(domSource, result);
144            return writer.toString();
145
146        } catch (TransformerException e) {
147            throw new IllegalStateException(e);
148        }
149    }
150
151    static Properties propertiesFromMap(Map<String, String> map) {
152        Properties props = new Properties();
153        props.putAll(map);
154        return props;
155    }
156
157    /** Canned default for HTML output. */
158    public static HashMap<String, String> OutputHtml() {
159        return methodMap("html");
160    }
161
162    /** Canned default for XML output. */
163    public static HashMap<String, String> OutputXml() {
164        return methodMap("xml");
165    }
166
167    private static HashMap<String, String> methodMap(String method) {
168        HashMap<String, String> map = new HashMap<>();
169        map.put(METHOD, method);
170        return map;
171    }
172
173    /**
174     * Convert a jsoup Document to a W3C Document. The created nodes will link back to the original
175     * jsoup nodes in the user property {@link #SourceProperty} (but after conversion, changes on one side will not
176     * flow to the other).
177     *
178     * @param in jsoup doc
179     * @return a W3C DOM Document representing the jsoup Document or Element contents.
180     */
181    public Document fromJsoup(org.jsoup.nodes.Document in) {
182        // just method API backcompat
183        return fromJsoup((org.jsoup.nodes.Element) in);
184    }
185
186    /**
187     * Convert a jsoup DOM to a W3C Document. The created nodes will link back to the original
188     * jsoup nodes in the user property {@link #SourceProperty} (but after conversion, changes on one side will not
189     * flow to the other). The input Element is used as a context node, but the whole surrounding jsoup Document is
190     * converted. (If you just want a subtree converted, use {@link #convert(org.jsoup.nodes.Element, Document)}.)
191     *
192     * @param in jsoup element or doc
193     * @return a W3C DOM Document representing the jsoup Document or Element contents.
194     * @see #sourceNodes(NodeList, Class)
195     * @see #contextNode(Document)
196     */
197    public Document fromJsoup(org.jsoup.nodes.Element in) {
198        Validate.notNull(in);
199        DocumentBuilder builder;
200        try {
201            builder = factory.newDocumentBuilder();
202            DOMImplementation impl = builder.getDOMImplementation();
203            Document out = builder.newDocument();
204            org.jsoup.nodes.Document inDoc = in.ownerDocument();
205            org.jsoup.nodes.DocumentType doctype = inDoc != null ? inDoc.documentType() : null;
206            if (doctype != null) {
207                try {
208                    org.w3c.dom.DocumentType documentType = impl.createDocumentType(doctype.name(), doctype.publicId(), doctype.systemId());
209                    out.appendChild(documentType);
210                } catch (DOMException ignored) {
211                    // invalid / empty doctype dropped
212                }
213            }
214            out.setXmlStandalone(true);
215            // if in is Document, use the root element, not the wrapping document, as the context:
216            org.jsoup.nodes.Element context = (in instanceof org.jsoup.nodes.Document) ? in.firstElementChild() : in;
217            out.setUserData(ContextProperty, context, null);
218            convert(inDoc != null ? inDoc : in, out);
219            return out;
220        } catch (ParserConfigurationException e) {
221            throw new IllegalStateException(e);
222        }
223    }
224
225    /**
226     * Converts a jsoup document into the provided W3C Document. If required, you can set options on the output
227     * document before converting.
228     *
229     * @param in jsoup doc
230     * @param out w3c doc
231     * @see org.jsoup.helper.W3CDom#fromJsoup(org.jsoup.nodes.Element)
232     */
233    public void convert(org.jsoup.nodes.Document in, Document out) {
234        // just provides method API backcompat
235        convert((org.jsoup.nodes.Element) in, out);
236    }
237
238    /**
239     * Converts a jsoup element into the provided W3C Document. If required, you can set options on the output
240     * document before converting.
241     *
242     * @param in jsoup element
243     * @param out w3c doc
244     * @see org.jsoup.helper.W3CDom#fromJsoup(org.jsoup.nodes.Element)
245     */
246    public void convert(org.jsoup.nodes.Element in, Document out) {
247        W3CBuilder builder = new W3CBuilder(out);
248        builder.namespaceAware = namespaceAware;
249        org.jsoup.nodes.Document inDoc = in.ownerDocument();
250        if (inDoc != null) {
251            if (!StringUtil.isBlank(inDoc.location())) {
252                out.setDocumentURI(inDoc.location());
253            }
254            builder.syntax = inDoc.outputSettings().syntax();
255        }
256        org.jsoup.nodes.Element rootEl = in instanceof org.jsoup.nodes.Document ? in.firstElementChild() : in; // skip the #root node if a Document
257        assert rootEl != null;
258        builder.traverse(rootEl);
259    }
260
261    /**
262     Evaluate an XPath query against the supplied document, and return the results.
263     @param xpath an XPath query
264     @param doc the document to evaluate against
265     @return the matches nodes
266     */
267    public NodeList selectXpath(String xpath, Document doc) {
268        return selectXpath(xpath, (Node) doc);
269    }
270
271    /**
272     Evaluate an XPath query against the supplied context node, and return the results.
273     @param xpath an XPath query
274     @param contextNode the context node to evaluate against
275     @return the matches nodes
276     */
277    public NodeList selectXpath(String xpath, Node contextNode) {
278        Validate.notEmptyParam(xpath, "xpath");
279        Validate.notNullParam(contextNode, "contextNode");
280
281        NodeList nodeList;
282        try {
283            // if there is a configured XPath factory, use that instead of the Java base impl:
284            String property = System.getProperty(XPathFactoryProperty);
285            final XPathFactory xPathFactory = property != null ?
286                XPathFactory.newInstance("jsoup") :
287                XPathFactory.newInstance();
288
289            XPathExpression expression = xPathFactory.newXPath().compile(xpath);
290            nodeList = (NodeList) expression.evaluate(contextNode, XPathConstants.NODESET); // love the strong typing here /s
291            Validate.notNull(nodeList);
292        } catch (XPathExpressionException | XPathFactoryConfigurationException e) {
293            throw new Selector.SelectorParseException(
294                e, "Could not evaluate XPath query [%s]: %s", xpath, e.getMessage());
295        }
296        return nodeList;
297    }
298
299    /**
300     Retrieves the original jsoup DOM nodes from a nodelist created by this convertor.
301     @param nodeList the W3C nodes to get the original jsoup nodes from
302     @param nodeType the jsoup node type to retrieve (e.g. Element, DataNode, etc)
303     @param <T> node type
304     @return a list of the original nodes
305     */
306    public <T extends org.jsoup.nodes.Node> List<T> sourceNodes(NodeList nodeList, Class<T> nodeType) {
307        Validate.notNull(nodeList);
308        Validate.notNull(nodeType);
309        List<T> nodes = new ArrayList<>(nodeList.getLength());
310
311        for (int i = 0; i < nodeList.getLength(); i++) {
312            org.w3c.dom.Node node = nodeList.item(i);
313            Object source = node.getUserData(W3CDom.SourceProperty);
314            if (nodeType.isInstance(source))
315                nodes.add(nodeType.cast(source));
316        }
317
318        return nodes;
319    }
320
321    /**
322     For a Document created by {@link #fromJsoup(org.jsoup.nodes.Element)}, retrieves the W3C context node.
323     @param wDoc Document created by this class
324     @return the corresponding W3C Node to the jsoup Element that was used as the creating context.
325     */
326    public Node contextNode(Document wDoc) {
327        return (Node) wDoc.getUserData(ContextNodeProperty);
328    }
329
330    /**
331     * Serialize a W3C document that was created by {@link #fromJsoup(org.jsoup.nodes.Element)} to a String.
332     * The output format will be XML or HTML depending on the content of the doc.
333     *
334     * @param doc Document
335     * @return Document as string
336     * @see W3CDom#asString(Document, Map)
337     */
338    public String asString(Document doc) {
339        return asString(doc, null);
340    }
341
342    /**
343     * Implements the conversion by walking the input.
344     */
345    protected static class W3CBuilder implements NodeVisitor {
346        private final Document doc;
347        private boolean namespaceAware = true;
348        private Node dest;
349        private Syntax syntax = Syntax.xml; // the syntax (to coerce attributes to). From the input doc if available.
350        /*@Nullable*/ private final org.jsoup.nodes.Element contextElement; // todo - unsure why this can't be marked nullable?
351
352        public W3CBuilder(Document doc) {
353            this.doc = doc;
354            dest = doc;
355            contextElement = (org.jsoup.nodes.Element) doc.getUserData(ContextProperty); // Track the context jsoup Element, so we can save the corresponding w3c element
356        }
357
358        @Override
359        public void head(org.jsoup.nodes.Node source, int depth) {
360            if (source instanceof org.jsoup.nodes.Element) {
361                org.jsoup.nodes.Element sourceEl = (org.jsoup.nodes.Element) source;
362                @Nullable String namespace = namespaceAware ? w3cNamespace(sourceEl) : null;
363                String tagName = Normalizer.xmlSafeTagName(sourceEl.tagName());
364                try {
365                    // use an empty namespace if none is present but the tag name has a prefix
366                    String imputedNamespace = namespace == null && tagName.contains(":") ? "" : namespace;
367                    Element el = doc.createElementNS(imputedNamespace, tagName);
368                    copyAttributes(sourceEl, el);
369                    append(el, sourceEl);
370                    if (sourceEl == contextElement)
371                        doc.setUserData(ContextNodeProperty, el, null);
372                    dest = el; // descend
373                } catch (DOMException e) {
374                    // If the Normalize didn't get it XML / W3C safe, inserts as plain text
375                    append(doc.createTextNode("<" + tagName + ">"), sourceEl);
376                }
377            } else if (source instanceof org.jsoup.nodes.TextNode) {
378                org.jsoup.nodes.TextNode sourceText = (org.jsoup.nodes.TextNode) source;
379                Text text = doc.createTextNode(sourceText.getWholeText());
380                append(text, sourceText);
381            } else if (source instanceof org.jsoup.nodes.Comment) {
382                org.jsoup.nodes.Comment sourceComment = (org.jsoup.nodes.Comment) source;
383                Comment comment = doc.createComment(sourceComment.getData());
384                append(comment, sourceComment);
385            } else if (source instanceof org.jsoup.nodes.DataNode) {
386                org.jsoup.nodes.DataNode sourceData = (org.jsoup.nodes.DataNode) source;
387                Text node = doc.createTextNode(sourceData.getWholeData());
388                append(node, sourceData);
389            } else {
390                // unhandled. note that doctype is not handled here - rather it is used in the initial doc creation
391            }
392        }
393
394        private static @Nullable String w3cNamespace(org.jsoup.nodes.Element sourceEl) {
395            // In W3C DOM, plain XML elements have no namespace; XML namespace is reserved for the {@code xml} prefix
396            String namespace = sourceEl.tag().namespace();
397            if (Parser.NamespaceXml.equals(namespace) && sourceEl.tag().prefix().isEmpty())
398                return null;
399            return namespace;
400        }
401
402        private void append(Node append, org.jsoup.nodes.Node source) {
403            append.setUserData(SourceProperty, source, null);
404            dest.appendChild(append);
405        }
406
407        @Override
408        public void tail(org.jsoup.nodes.Node source, int depth) {
409            if (source instanceof org.jsoup.nodes.Element && dest.getParentNode() instanceof Element) {
410                dest = dest.getParentNode(); // undescend
411            }
412        }
413
414        private void copyAttributes(org.jsoup.nodes.Element jEl, Element wEl) {
415            for (Attribute attribute : jEl.attributes()) {
416                try {
417                    setAttribute(jEl, wEl, attribute, syntax);
418                } catch (DOMException e) {
419                    if (syntax != Syntax.xml)
420                        setAttribute(jEl, wEl, attribute, Syntax.xml);
421                }
422            }
423        }
424
425        private void setAttribute(org.jsoup.nodes.Element jEl, Element wEl, Attribute attribute, Syntax syntax) throws DOMException {
426            String key = Attribute.getValidKey(attribute.getKey(), syntax);
427            if (key != null) {
428                String namespace = attribute.namespace();
429                if (namespaceAware && !namespace.isEmpty())
430                    wEl.setAttributeNS(namespace, key, attribute.getValue());
431                else
432                    wEl.setAttribute(key, attribute.getValue());
433                maybeAddUndeclaredNs(namespace, key, jEl, wEl);
434            }
435        }
436
437        /**
438         Add a namespace declaration for an attribute with a prefix if it is not already present. Ensures that attributes
439         with prefixes have the corresponding namespace declared, E.g. attribute "v-bind:foo" gets another attribute
440         "xmlns:v-bind='undefined'. So that the asString() transformation pass is valid.
441         If the parser was HTML we don't have a discovered namespace but we are trying to coerce it, so walk up the
442         element stack and find it.
443         */
444        private void maybeAddUndeclaredNs(String namespace, String attrKey, org.jsoup.nodes.Element jEl, Element wEl) {
445            if (!namespaceAware || !namespace.isEmpty()) return;
446            int pos = attrKey.indexOf(':');
447            if (pos != -1) { // prefixed but no namespace defined during parse, add a fake so that w3c serialization doesn't blow up
448                String prefix = attrKey.substring(0, pos);
449                if (prefix.equals("xmlns")) return;
450                org.jsoup.nodes.Document doc = jEl.ownerDocument();
451                if (doc != null && doc.parser().getTreeBuilder() instanceof HtmlTreeBuilder) {
452                    // try walking up the stack and seeing if there is a namespace declared for this prefix (and that we didn't parse because HTML)
453                    for (org.jsoup.nodes.Element el = jEl; el != null; el = el.parent()) {
454                        String ns = el.attr("xmlns:" + prefix);
455                        if (!ns.isEmpty()) {
456                            namespace = ns;
457                            // found it, set it
458                            wEl.setAttributeNS(namespace, attrKey, jEl.attr(attrKey));
459                            return;
460                        }
461                    }
462                }
463
464                // otherwise, put in a fake one
465                wEl.setAttribute("xmlns:" + prefix, undefinedNs);
466            }
467        }
468        private static final String undefinedNs = "undefined";
469    }
470
471}