List of usage examples for java.net URL getDefaultPort
public int getDefaultPort()
From source file:crawler.java.edu.uci.ics.crawler4j.robotstxt.RobotstxtServer.java
private HostDirectives fetchDirectives(URL url) { WebURL robotsTxtUrl = new WebURL(); String host = getHost(url);/*w w w . jav a 2 s . c om*/ String port = ((url.getPort() == url.getDefaultPort()) || (url.getPort() == -1)) ? "" : (":" + url.getPort()); robotsTxtUrl.setURL("http://" + host + port + "/robots.txt"); HostDirectives directives = null; PageFetchResult fetchResult = null; try { fetchResult = pageFetcher.fetchPage(robotsTxtUrl); if (fetchResult.getStatusCode() == HttpStatus.SC_OK) { Page page = new Page(robotsTxtUrl); fetchResult.fetchContent(page); if (Util.hasPlainTextContent(page.getContentType())) { String content; if (page.getContentCharset() == null) { content = new String(page.getContentData()); } else { content = new String(page.getContentData(), page.getContentCharset()); } directives = RobotstxtParser.parse(content, config.getUserAgentName()); } else if (page.getContentType().contains("html")) { // TODO This one should be upgraded to remove all html tags String content = new String(page.getContentData()); directives = RobotstxtParser.parse(content, config.getUserAgentName()); } else { logger.warn( "Can't read this robots.txt: {} as it is not written in plain text, contentType: {}", robotsTxtUrl.getURL(), page.getContentType()); } } else { logger.debug("Can't read this robots.txt: {} as it's status code is {}", robotsTxtUrl.getURL(), fetchResult.getStatusCode()); } } catch (SocketException | UnknownHostException | SocketTimeoutException | NoHttpResponseException se) { // No logging here, as it just means that robots.txt doesn't exist on this server which is perfectly ok } catch (PageBiggerThanMaxSizeException pbtms) { logger.error("Error occurred while fetching (robots) url: {}, {}", robotsTxtUrl.getURL(), pbtms.getMessage()); } catch (Exception e) { logger.error("Error occurred while fetching (robots) url: " + robotsTxtUrl.getURL(), e); } finally { if (fetchResult != null) { fetchResult.discardContentIfNotConsumed(); } } if (directives == null) { // We still need to have this object to keep track of the time we fetched it directives = new HostDirectives(); } synchronized (host2directivesCache) { if (host2directivesCache.size() == config.getCacheSize()) { String minHost = null; long minAccessTime = Long.MAX_VALUE; for (Map.Entry<String, HostDirectives> entry : host2directivesCache.entrySet()) { if (entry.getValue().getLastAccessTime() < minAccessTime) { minAccessTime = entry.getValue().getLastAccessTime(); minHost = entry.getKey(); } } host2directivesCache.remove(minHost); } host2directivesCache.put(host, directives); } return directives; }
From source file:de.ecclesia.kipeto.RepositoryResolver.java
/** * Ermittelt anhand der Default-Repository-URL die IP-Adresse der * Netzwerkkarte, die Verbindung zum Repository hat. *//*w ww . jav a2 s. com*/ private String determinateLocalIP() throws IOException { Socket socket = null; try { int port; String hostname; if (isSftp()) { port = 22; hostname = getHostname(); } else { URL url = new URL(defaultRepositoryUrl); port = url.getPort() > -1 ? url.getPort() : url.getDefaultPort(); hostname = url.getHost(); } log.debug("Determinating local IP-Adress by connect to {}:{}", defaultRepositoryUrl, port); InetAddress address = Inet4Address.getByName(hostname); socket = new Socket(); socket.connect(new InetSocketAddress(address, port), 3000); InputStream stream = socket.getInputStream(); InetAddress localAddress = socket.getLocalAddress(); stream.close(); String localIp = localAddress.getHostAddress(); log.info("Local IP-Adress is {}", localIp); return localIp; } finally { if (socket != null) { socket.close(); } } }
From source file:crawler.RobotstxtServer.java
private HostDirectives fetchDirectives(URL url) { WebURL robotsTxtUrl = new WebURL(); String host = getHost(url);// w ww . j a v a 2s. c o m String port = ((url.getPort() == url.getDefaultPort()) || (url.getPort() == -1)) ? "" : (":" + url.getPort()); String proto = url.getProtocol(); robotsTxtUrl.setURL(proto + "://" + host + port + "/robots.txt"); HostDirectives directives = null; PageFetchResult fetchResult = null; try { for (int redir = 0; redir < 3; ++redir) { fetchResult = pageFetcher.fetchPage(robotsTxtUrl); int status = fetchResult.getStatusCode(); // Follow redirects up to 3 levels if ((status == HttpStatus.SC_MULTIPLE_CHOICES || status == HttpStatus.SC_MOVED_PERMANENTLY || status == HttpStatus.SC_MOVED_TEMPORARILY || status == HttpStatus.SC_SEE_OTHER || status == HttpStatus.SC_TEMPORARY_REDIRECT || status == 308) && // SC_PERMANENT_REDIRECT RFC7538 fetchResult.getMovedToUrl() != null) { robotsTxtUrl.setURL(fetchResult.getMovedToUrl()); fetchResult.discardContentIfNotConsumed(); } else { // Done on all other occasions break; } } if (fetchResult.getStatusCode() == HttpStatus.SC_OK) { Page page = new Page(robotsTxtUrl); // Most recent answer on robots.txt max size is // https://goo.gl/OqpKbP fetchResult.fetchContent(page, 10_000 * 1024); if (Util.hasPlainTextContent(page.getContentType())) { String content; if (page.getContentCharset() == null) { content = new String(page.getContentData()); } else { content = new String(page.getContentData(), page.getContentCharset()); } directives = RobotstxtParser.parse(content, config); } else if (page.getContentType().contains("html")) { // TODO This one should be upgraded to remove all // html tags String content = new String(page.getContentData()); directives = RobotstxtParser.parse(content, config); } else { logger.warn("Can't read this robots.txt: {} as it is not written in plain text, " + "contentType: {}", robotsTxtUrl.getURL(), page.getContentType()); } } else { logger.debug("Can't read this robots.txt: {} as it's status code is {}", robotsTxtUrl.getURL(), fetchResult.getStatusCode()); } } catch (SocketException | UnknownHostException | SocketTimeoutException | NoHttpResponseException se) { // No logging here, as it just means that robots.txt doesn't exist on this server // which is perfectly ok logger.trace("robots.txt probably does not exist.", se); } catch (PageBiggerThanMaxSizeException pbtms) { logger.error("Error occurred while fetching (robots) url: {}, {}", robotsTxtUrl.getURL(), pbtms.getMessage()); } catch (Exception e) { logger.error("Error occurred while fetching (robots) url: " + robotsTxtUrl.getURL(), e); } finally { if (fetchResult != null) { fetchResult.discardContentIfNotConsumed(); } } if (directives == null) { // We still need to have this object to keep track of the time we fetched it directives = new HostDirectives(config); } synchronized (host2directivesCache) { if (host2directivesCache.size() == config.getCacheSize()) { String minHost = null; long minAccessTime = Long.MAX_VALUE; for (Map.Entry<String, HostDirectives> entry : host2directivesCache.entrySet()) { long entryAccessTime = entry.getValue().getLastAccessTime(); if (entryAccessTime < minAccessTime) { minAccessTime = entryAccessTime; minHost = entry.getKey(); } } host2directivesCache.remove(minHost); } host2directivesCache.put(host, directives); } return directives; }
From source file:org.commonjava.indy.httprox.util.ProxyResponseHelper.java
private int getPort(URL url) { int port = url.getPort(); if (port < 1) { port = url.getDefaultPort(); }//from ww w. jav a 2 s . c om return port; }
From source file:savant.thousandgenomes.FTPBrowser.java
private void setRoot(URL rootURL) { host = rootURL.getHost();/*from w w w . j a v a2 s. co m*/ int p = rootURL.getPort(); port = p != -1 ? p : rootURL.getDefaultPort(); rootDir = new File(rootURL.getPath()); curDir = rootDir; }
From source file:org.warlock.spine.connection.Transmitter.java
@Override public void run() { ConnectionManager c = ConnectionManager.getInstance(); if (!sendable.recordTry()) { if (sendable.getMessageId() != null) { c.removeRequest(sendable.getMessageId()); sendable.expire();/*from w w w .j a v a 2 s .c o m*/ } return; } SpineSecurityContext tlsContext = c.getSecurityContext(); // SSLSocketFactory sf = tlsContext.getSocketFactory(); String h = sendable.getResolvedUrl(); String host = null; int port = 443; try { if (h == null) { // Retry of persisted reliable message from previous MHS session // host = ((org.warlock.spine.messaging.EbXmlMessage) sendable).getHost(); } else { sendable.persist(); URL u = new URL(h); host = u.getHost(); port = (u.getPort() == -1) ? u.getDefaultPort() : u.getPort(); } //Override host and port when using Proxy String proxyhost = System.getProperty(PROXYHOST); if (proxyhost != null && (proxyhost.trim().length() != 0)) { host = proxyhost; } String p = System.getProperty(PROXYPORT); if ((p != null) && (p.trim().length() != 0)) { try { int proxyport = Integer.parseInt(p); port = proxyport; } catch (NumberFormatException e) { System.err.println("Asynchronous wait period not a valid integer - " + e.toString()); } } try (Socket s = tlsContext.createSocket(host, port)) { int replyLength = -1; SessionCaptor sc = c.getSessionCaptor(); if (sc == null) { sendable.write(s.getOutputStream()); replyLength = getHeader(s.getInputStream()); if (replyLength == -1) { SpineToolsLogger.getInstance().log("org.warlock.spine.connection.Transmitter.noResponse", "Could not read response sending " + sendable.getMessageId()); s.close(); return; } if (replyLength > 0) readSynchronousReply(s.getInputStream(), replyLength); } else { ByteArrayOutputStream outStream = new ByteArrayOutputStream(); TeeOutputStream tos = new TeeOutputStream(s.getOutputStream(), outStream); sendable.write(tos); sendable.setOnTheWireRequest(outStream.toByteArray()); ByteArrayOutputStream inStream = new ByteArrayOutputStream(); TeeInputStream tis = new TeeInputStream(s.getInputStream(), inStream); replyLength = getHeader(tis); if (replyLength == -1) { SpineToolsLogger.getInstance().log("org.warlock.spine.connection.Transmitter.noResponse", "Could not read response sending " + sendable.getMessageId()); s.close(); sc.capture(sendable); return; } if (replyLength > 0) { readSynchronousReply(tis, replyLength); sendable.setOnTheWireResponse(inStream.toByteArray()); } sc.capture(sendable); } } if (sendable.getType() == Sendable.SOAP) { if (sendable.getSynchronousResponse() == null) { SpineToolsLogger.getInstance().log( "org.warlock.spine.connection.Transmitter.noResponseReceived", "No response to " + sendable.getMessageId()); return; } SynchronousResponseHandler handler = c.getSynchronousResponseHandler(sendable.getSoapAction()); handler.handle((SpineSOAPRequest) sendable); return; } if (sendable.getMessageId() != null) { // Don't do this for asynchronous acks if (sendable.getSynchronousResponse() != null) { if ((responseHeader != null) && (responseHeader.contains("HTTP 5"))) { SpineToolsLogger.getInstance().log( "org.warlock.spine.connection.Transmitter.HTTP500received", "HTTP 500 received sending " + sendable.getMessageId()); c.removeRequest(sendable.getMessageId()); } else { if (sendable.getSynchronousResponse().contains(sendable.getMessageId())) { c.registerAck(sendable.getMessageId()); } if (sendable.getSynchronousResponse().contains("Bad request")) { c.registerAck(sendable.getMessageId()); SpineToolsLogger.getInstance().log( "org.warlock.spine.connection.Transmitter.HTTP500received", "Bad request received sending " + sendable.getMessageId()); } } } } } catch (Exception eIo) { SpineToolsLogger.getInstance().log("org.warlock.spine.connection.Transmitter.IOException", "IOException sending " + sendable.getMessageId() + eIo.getMessage()); } }
From source file:org.jasig.portlet.calendar.adapter.CalDavCalendarAdapter.java
protected final net.fortuna.ical4j.model.Calendar retrieveCalendar(String url, Interval interval, Credentials credentials) {//from w ww . j a va2s . co m try { // construct a HostConfiguration from the server URL URL hostUrl = new URL(url); int port = hostUrl.getPort(); if (port == -1) { port = hostUrl.getDefaultPort(); } HttpClient httpClient = new HttpClient(); httpClient.getHostConfiguration().setHost(hostUrl.getHost(), port, Protocol.getProtocol(hostUrl.getProtocol())); if (log.isDebugEnabled()) { log.debug("connecting to calDAV host " + httpClient.getHostConfiguration().getHost()); } if (credentials != null) { httpClient.getState().setCredentials(AuthScope.ANY, credentials); httpClient.getParams().setAuthenticationPreemptive(true); } String calendarPath = hostUrl.getPath(); CalDAVCollection calDAVCollection = new CalDAVCollection(calendarPath, httpClient.getHostConfiguration(), new CalDAV4JMethodFactory(), CalDAVConstants.PROC_ID_DEFAULT); // log.debug(calDAVCollection.testConnection(httpClient)); if (log.isDebugEnabled()) { log.debug(calDAVCollection.getCalendarCollectionRoot()); } // Haven't gotten queries working // GenerateQuery gq = new GenerateQuery(); // gq.setTimeRange(new net.fortuna.ical4j.model.Date(interval // .getStart().toDate()), new net.fortuna.ical4j.model.Date( // interval.getEnd().toDate())); // CalendarQuery query = gq.generate(); // Haven't gotten queries working even for single calendar. // Also I don't think Google supports viewing // or querying multiple calendars at once. // List<Calendar> cals = calDAVCollection.queryCalendars(client, query); // return cals.get(0); Calendar cal = calDAVCollection.getCalendar(httpClient, ""); if (log.isDebugEnabled()) { log.debug(cal); } return cal; } catch (CalDAV4JException e) { log.error("CalDAV exception: ", e); throw new CalendarException(e); } catch (Exception e) { log.error(e); throw new CalendarException("Unknown exception while retrieving calendar", e); } }
From source file:com.nginious.http.plugin.RollbackState.java
void checkState(IProject project) { try {//from w ww . j a v a 2 s .c om String publishUrl = project.getPersistentProperty(NginiousPlugin.PUBLISH_URL_PROP_KEY); if (publishUrl == null) { publishUrl = NginiousPlugin.DEFAULT_PUBLISH_URL; } String publishUsername = project.getPersistentProperty(NginiousPlugin.PUBLISH_USERNAME_PROP_KEY); if (publishUsername == null) { publishUsername = NginiousPlugin.DEFAULT_PUBLISH_USERNAME; } String publishPassword = project.getPersistentProperty(NginiousPlugin.PUBLISH_PASSWORD_PROP_KEY); if (publishPassword == null) { publishPassword = NginiousPlugin.DEFAULT_PUBLISH_PASSWORD; } URL url = new URL(publishUrl); int port = url.getPort(); if (port == -1) { port = url.getDefaultPort(); } HttpClientRequest request = new HttpClientRequest(); request.setMethod(HttpMethod.GET); request.setPath(url.getPath()); request.setHeader("Host", url.getHost()); request.setHeader("Content-Type", "text/xml; charset=utf-8"); request.setHeader("Connection", "close"); request.setHeader("Content-Length", "0"); String authorization = createAuthorization(HttpMethod.GET, publishUsername, publishPassword); request.setHeader("Authorization", authorization); HttpClient client = new HttpClient(url.getHost(), port); HttpClientResponse response = client.request(request, "".getBytes()); if (response.getStatus() == HttpStatus.OK) { setState(project, response); } } catch (HttpClientException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (CoreException e) { e.printStackTrace(); } }
From source file:com.predic8.membrane.core.interceptor.rewrite.ReverseProxyingInterceptor.java
/** * handles "Destination" header (see RFC 2518 section 9.3; also used by WebDAV) *//* w w w.j av a2 s. c o m*/ @Override public Outcome handleRequest(Exchange exc) throws Exception { if (exc.getRequest() == null) return Outcome.CONTINUE; String destination = exc.getRequest().getHeader().getFirstValue(Header.DESTINATION); if (destination == null) return Outcome.CONTINUE; if (!destination.contains("://")) return Outcome.CONTINUE; // local redirect (illegal by spec) // do not rewrite, if the client does not refer to the same host if (!isSameSchemeHostAndPort(getProtocol(exc) + "://" + exc.getRequest().getHeader().getHost(), destination)) return Outcome.CONTINUE; // if we cannot determine the target hostname if (exc.getDestinations().isEmpty()) { // just remove the schema/hostname/port. this is illegal (by the spec), // but most clients understand it exc.getRequest().getHeader().setValue(Header.DESTINATION, new URL(destination).getFile()); return Outcome.CONTINUE; } URL target = new URL(exc.getDestinations().get(0)); // rewrite to our schema, host and port exc.getRequest().getHeader().setValue(Header.DESTINATION, Relocator.getNewLocation(destination, target.getProtocol(), target.getHost(), target.getPort() == -1 ? target.getDefaultPort() : target.getPort())); return Outcome.CONTINUE; }
From source file:com.devicehive.application.filter.SwaggerFilter.java
@Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { URL requestUrl = new URL(request.getRequestURL().toString()); logger.debug("Swagger filter triggered by '{}: {}'. Request will be redirected to swagger page", request.getMethod(), requestUrl); String swaggerJsonUrl = String.format("%s://%s:%s%s%s/swagger.json", requestUrl.getProtocol(), requestUrl.getHost(),/*from w w w .ja v a2 s. com*/ requestUrl.getPort() == -1 ? requestUrl.getDefaultPort() : requestUrl.getPort(), request.getContextPath(), JerseyConfig.REST_PATH); String url = request.getContextPath() + "/swagger.html?url=" + swaggerJsonUrl; logger.debug("Request is being redirected to '{}'", url); response.sendRedirect(url); }