Java tutorial
/** * License Agreement for OpenSearchServer * * Copyright (C) 2012-2013 Emmanuel Keller / Jaeksoft * * http://www.open-search-server.com * * This file is part of OpenSearchServer. * * OpenSearchServer is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * OpenSearchServer is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with OpenSearchServer. * If not, see <http://www.gnu.org/licenses/>. **/ package com.jaeksoft.searchlib.parser.htmlParser; import java.io.IOException; import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.Collection; import java.util.List; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathExpressionException; import org.apache.commons.lang3.StringEscapeUtils; import org.xml.sax.SAXException; import com.jaeksoft.searchlib.Logging; import com.jaeksoft.searchlib.SearchLibException; import com.jaeksoft.searchlib.streamlimiter.StreamLimiter; import com.jaeksoft.searchlib.util.LinkUtils; import com.jaeksoft.searchlib.util.MimeUtils; public abstract class HtmlDocumentProvider { public static interface XPath { public abstract void xPath(String xPath, Collection<Object> nodes) throws XPathExpressionException; } private final HtmlParserEnum parserEnum; private String titleCache; private List<HtmlNodeAbstract<?>> metasCache; private HtmlNodeAbstract<?> rootNode; private int score; protected HtmlDocumentProvider(HtmlParserEnum parserEnum) { this.parserEnum = parserEnum; titleCache = null; metasCache = null; score = 0; rootNode = null; } public void init(String charset, StreamLimiter streamLimiter) throws SAXException, IOException, ParserConfigurationException, SearchLibException { rootNode = getDocument(charset, streamLimiter); } public void init(String htmlSource) throws IOException, ParserConfigurationException, SAXException { rootNode = getDocument(htmlSource); } public HtmlNodeAbstract<?> getRootNode() { return rootNode; } public final String getName() { return parserEnum.getLabel(); } protected abstract HtmlNodeAbstract<?> getDocument(String charset, InputStream inputStream) throws SAXException, IOException, ParserConfigurationException; protected HtmlNodeAbstract<?> getDocument(String charset, StreamLimiter streamLimiter) throws SAXException, IOException, ParserConfigurationException, SearchLibException { return getDocument(charset, streamLimiter.getNewInputStream()); } protected abstract HtmlNodeAbstract<?> getDocument(String htmlSource) throws IOException, ParserConfigurationException, SAXException; public void score() { score = getTitle() != null ? 10000 : 0; score += getMetas() != null ? metasCache.size() * 1000 : 0; score += rootNode != null ? rootNode.countElements() : 0; } final public String getTitle() { if (titleCache != null) return titleCache; if (rootNode == null) return null; String[] p1 = { "html", "head", "title" }; String title = rootNode.getFirstTextNode(p1); if (title == null) { String[] p2 = { "html", "title" }; title = rootNode.getFirstTextNode(p2); } if (title == null) return null; titleCache = StringEscapeUtils.unescapeHtml4(title); return titleCache; } final public URL getCanonicalLink(URL currentUrl) { if (rootNode == null) return null; String[] p1 = { "html", "head", "link" }; List<HtmlNodeAbstract<?>> nodes = rootNode.getNodes(p1); if (nodes == null) return null; for (HtmlNodeAbstract<?> node : nodes) { String rel = node.getAttribute("rel"); if (rel == null) continue; if (!"canonical".equalsIgnoreCase(rel)) continue; String href = node.getAttribute("href"); if (href == null) return null; return LinkUtils.getLink(currentUrl, href, null, false); } return null; } final public List<HtmlNodeAbstract<?>> getMetas() { if (metasCache != null) return metasCache; if (rootNode == null) return null; final String[] p1 = { "html", "head", "meta" }; final String[] p2 = { "html", "meta" }; metasCache = rootNode.getNewNodeList(); rootNode.getNodes(metasCache, p1); rootNode.getNodes(metasCache, p2); return metasCache; } final public static String getMetaContent(final HtmlNodeAbstract<?> node) { String content = node.getAttributeText("content"); if (content == null) return null; return StringEscapeUtils.unescapeHtml4(content); } final public String getMetaHttpEquiv(String name) { getMetas(); if (metasCache == null) return null; for (HtmlNodeAbstract<?> node : metasCache) { String attr_http_equiv = node.getAttributeText("http-equiv"); if (name.equalsIgnoreCase(attr_http_equiv)) return getMetaContent(node); } return null; } final public String getMetaCharset() { String contentType = getMetaHttpEquiv("content-type"); if (contentType == null) return null; return MimeUtils.extractContentTypeCharset(contentType); } final public URL getBaseHref() { List<HtmlNodeAbstract<?>> list = rootNode.getNodes("html", "head", "base"); if (list == null) return null; if (list.size() == 0) return null; HtmlNodeAbstract<?> node = list.get(0); if (node == null) return null; String url = node.getAttributeText("href"); if (url == null) return null; try { return LinkUtils.newEncodedURL(url); } catch (MalformedURLException e) { Logging.warn(e); return null; } catch (URISyntaxException e) { Logging.warn(e); return null; } } final public static HtmlDocumentProvider bestScore(List<HtmlDocumentProvider> providers) { HtmlDocumentProvider bestProvider = null; for (HtmlDocumentProvider provider : providers) { provider.score(); if (bestProvider == null) bestProvider = provider; else if (provider.score > bestProvider.score) bestProvider = provider; } return bestProvider; } public abstract boolean isXPathSupported(); public void xPath(String xPath, Collection<Object> nodes) throws XPathExpressionException { ((XPath) rootNode).xPath(xPath, nodes); } }