Java tutorial
/* * Copyright (c) 2009, MediaEvent Services GmbH & Co. KG * http://mediaeventservices.com * * This file is part of Marbles. * * Marbles is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Marbles is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Marbles. If not, see <http://www.gnu.org/licenses/>. * */ package de.fuberlin.wiwiss.marbles.loading; import java.util.ArrayList; import java.util.Collection; import java.util.List; import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.URI; import org.apache.commons.httpclient.URIException; import org.apache.commons.httpclient.params.HttpConnectionManagerParams; import org.openrdf.model.BNode; import org.openrdf.model.Resource; import org.openrdf.repository.Repository; import de.fuberlin.wiwiss.marbles.dataproviders.DataProvider; import de.fuberlin.wiwiss.marbles.dataproviders.RevyuProvider; import de.fuberlin.wiwiss.marbles.dataproviders.SindiceProvider; /** * Provides functionalities to load URLs and to discover related data by means of data providers. * * @author Christian Becker */ public class SemanticWebClient { /** * Number of seconds to wait to load an URL */ final int CONNECTION_TIMEOUT = 20; /** * Number of seconds to wait for additional data once focal resource is loaded */ final int TIME_LIMIT_ADDITIONAL = 3; /** * Maximum number of steps for autonomous discovery */ final int MAX_STEPS = 1; /** * Maximum number of redirects to follow in the course of a single request for a document */ final int MAX_REDIRECTS = 2; private Collection<DataProvider> dataProviders; private DereferencingTaskQueue uriQueue; private HttpClient httpClient; private CacheController cacheController; /** * Constructs a new <code>SemanticWebClient</code> * * @param cacheController * @param spongerProvider * @param dataProviders */ public SemanticWebClient(CacheController cacheController, SpongerProvider spongerProvider, Collection<DataProvider> dataProviders) { this.cacheController = cacheController; this.dataProviders = dataProviders; /* Set connection parameters */ HttpConnectionManagerParams httpManagerParams = new HttpConnectionManagerParams(); httpManagerParams.setConnectionTimeout(CONNECTION_TIMEOUT * 1000); httpManagerParams.setTcpNoDelay(true); httpManagerParams.setStaleCheckingEnabled(true); MultiThreadedHttpConnectionManager httpManager = new MultiThreadedHttpConnectionManager(); httpManager.setParams(httpManagerParams); httpClient = new HttpClient(httpManager); uriQueue = new DereferencingTaskQueue(httpClient, spongerProvider, 10 /* maxThreads */, 500 * 1024 /* maxFileSize */); } /** * Builds list of URLs to be loaded to learn more about a resource. * Uses data providers. * * @param resource */ private List<URI> getURLsForResource(Resource resource) { List<URI> urls = new ArrayList<URI>(); /* Dereference the resource itself */ try { if (!(resource instanceof BNode)) { urls.add(new URI(resource.toString(), true)); /* Temporarily work around DBpedia 303 redirection bug that occurs when special characters are involved */ if (resource.toString().startsWith("http://dbpedia.org/resource/")) { urls.add(new URI( resource.toString().replace("http://dbpedia.org/resource/", "http://dbpedia.org/data/") + ".xml", true)); } } } catch (URIException e) { e.printStackTrace(); } catch (NullPointerException e) { e.printStackTrace(); } /* and ask the data providers */ for (DataProvider provider : dataProviders) { URI queryURL; if (null != (queryURL = provider.getQueryURL(resource))) urls.add(queryURL); } return urls; } /** * Initiates a {@link DereferencerBatch} to retrieve data for a given resource * * @param resource * @return List of URLs queries in the process; these may be looked up in the metadata store for details */ public List<URI> discoverResource(Resource resource, boolean wait) { List<URI> urlsToBeFetched = getURLsForResource(resource); DereferencerBatch dereferencerBatch = new DereferencerBatch(cacheController, uriQueue, dataProviders, resource, MAX_STEPS, MAX_REDIRECTS); /* provide URLs to dereferencer */ for (URI url : urlsToBeFetched) { try { dereferencerBatch.loadURL(url, 0 /* step */, 0 /* redirect step */, false /* don't force reload */); } catch (URIException e) { e.printStackTrace(); } } /* Initiate link retrieval from any previous data */ dereferencerBatch.processLinks(1); /* Wait loop with timeout */ long timeStarted = System.currentTimeMillis(); System.err.println(Thread.currentThread().getName() + ": starting discoverResource() at " + timeStarted); if (wait) { synchronized (dereferencerBatch) { while (dereferencerBatch.hasPending(0) || ((System.currentTimeMillis() - timeStarted < TIME_LIMIT_ADDITIONAL * 1000) && dereferencerBatch.hasPending())) { try { dereferencerBatch.wait(100); } catch (InterruptedException e) { } } } } System.err.println(Thread.currentThread().getName() + ": finished discoverResource() after " + ((System.currentTimeMillis() - timeStarted) / 1000) + "s"); /* * We stop waiting here so that the data retrieved so far can be shown to the client. * Nonetheless, retrieval is not canceled - the client could refresh at a later time to get it * (AJAX automation would make a lot of sense here), and additional information can be incorporated * into subsequent views */ return dereferencerBatch.getRetrievedURLs(); } /** * Loads a given URL into the cache using a {@link DereferencerBatch} * * @param url The URL to be loaded * @param wait If true, the method returns after the request has been processed */ public void loadURL(URI url, boolean wait) { DereferencerBatch dereferencerBatch = new DereferencerBatch(cacheController, uriQueue, dataProviders, null, 0 /* maxSteps (!) */, MAX_REDIRECTS); /* Provide URLs to dereferencer */ try { dereferencerBatch.loadURL(url, 0 /* step */, 0 /* redirect step */, true /* force reload */); } catch (URIException e1) { e1.printStackTrace(); return; } if (wait) { synchronized (dereferencerBatch) { while (dereferencerBatch.hasPending()) { try { dereferencerBatch.wait(100); } catch (InterruptedException e) { } } } } } }