List of usage examples for java.net URLConnection getContentType
public String getContentType()
From source file:org.eclipse.smarthome.binding.sonos.handler.ZonePlayerHandler.java
private String getContentTypeFromUrl(URL url) { if (url == null) { return null; }// w ww .j a va 2 s . c om String contentType; InputStream input = null; try { URLConnection connection = url.openConnection(); contentType = connection.getContentType(); logger.debug("Content type from headers: {}", contentType); if (contentType == null) { input = connection.getInputStream(); contentType = URLConnection.guessContentTypeFromStream(input); logger.debug("Content type from data: {}", contentType); if (contentType == null) { contentType = RawType.DEFAULT_MIME_TYPE; } } } catch (IOException e) { logger.debug("Failed to identify content type from URL: {}", e.getMessage()); contentType = RawType.DEFAULT_MIME_TYPE; } finally { IOUtils.closeQuietly(input); } return contentType; }
From source file:CB_Utils.Util.Downloader.java
/** * Start downloading the remote resource. The target object should not be accessed until after calling waitUntilCompleted(). *///from w ww.j ava 2s . c o m @Override public void run() { synchronized (stateLock) { if (started) { return; } else { started = true; running = true; } } BufferedInputStream bis = null; BufferedOutputStream bos = null; BufferedReader br = null; try { /* open connection to the URL */ checkState(); progressString = "Opening connection to remote resource"; progressUpdated = true; final URLConnection link; try { link = url.openConnection(); link.connect(); } catch (Exception e) { progressString = "Failed to open connection to remote resource"; progressUpdated = true; throw e; } /* get length of the remote resource */ checkState(); progressString = "Getting length of remote resource"; progressUpdated = true; /* get size of webpage in bytes; -1 if unknown */ final int length = link.getContentLength(); synchronized (lengthLock) { totalLength = length; } progressUpdated = true; /* open input stream to remote resource */ checkState(); progressString = "Opening input stream to remote resource"; progressUpdated = true; InputStream input = null; try { if (totalLength < 1) { // load with http Request HttpGet httppost = new HttpGet(url.toString()); // Execute HTTP Post Request try { HttpParams httpParameters = new BasicHttpParams(); HttpConnectionParams.setConnectionTimeout(httpParameters, HttpUtils.conectionTimeout); HttpConnectionParams.setSoTimeout(httpParameters, HttpUtils.socketTimeout); DefaultHttpClient httpClient = new DefaultHttpClient(httpParameters); HttpResponse response = httpClient.execute(httppost); input = response.getEntity().getContent(); } catch (ConnectTimeoutException e1) { e1.printStackTrace(); } catch (ClientProtocolException e1) { e1.printStackTrace(); } catch (IOException e1) { e1.printStackTrace(); } } else { input = link.getInputStream(); } if (target instanceof File) { bis = new BufferedInputStream(input); } else if (target instanceof StringBuilder) { final String contentType = link.getContentType().toLowerCase(Locale.ENGLISH); /* look for charset, if specified */ String charset = null; final Matcher m = Pattern.compile(".*charset[\\s]*=([^;]++).*").matcher(contentType); if (m.find()) { charset = m.group(1).trim(); } if ((charset != null) && charset.length() > 0) { try { br = new BufferedReader(new InputStreamReader(input, charset)); } catch (Exception e) { br = null; } } if (br == null) { br = new BufferedReader(new InputStreamReader(input)); } } } catch (Exception e) { progressString = "Failed to open input stream to remote resource"; progressUpdated = true; throw e; } /* open output stream, if necessary */ if (target instanceof File) { checkState(); progressString = "Opening output stream to local file"; progressUpdated = true; try { /* create parent directories, if necessary */ final File f = (File) target; final File parent = f.getParentFile(); if ((parent != null) && !parent.exists()) { parent.mkdirs(); } bos = new BufferedOutputStream(f.getFileOutputStream()); } catch (Exception e) { progressString = "Failed to open output stream to local file"; progressUpdated = true; throw e; } } /* download remote resource iteratively */ progressString = "Downloading"; progressUpdated = true; try { if (target instanceof File) { final byte[] byteBuffer = new byte[BUFFER_SIZE]; while (true) { checkState(); final int byteCount = bis.read(byteBuffer, 0, BUFFER_SIZE); /* check for end-of-stream */ if (byteCount == -1) { break; } bos.write(byteBuffer, 0, byteCount); synchronized (lengthLock) { downloadedLength += byteCount; } progressUpdated = true; } } else if (target instanceof StringBuilder) { final char[] charBuffer = new char[BUFFER_SIZE]; final StringBuilder sb = (StringBuilder) target; while (true) { checkState(); final int charCount = br.read(charBuffer, 0, BUFFER_SIZE); /* check for end-of-stream */ if (charCount == -1) { break; } sb.append(charBuffer, 0, charCount); synchronized (lengthLock) { downloadedLength += charCount; /* may be inaccurate because byte != char */ } progressUpdated = true; } } } catch (Exception e) { progressString = "Failed to download remote resource"; progressUpdated = true; throw e; } /* download completed successfully */ progressString = "Download completed"; progressUpdated = true; } catch (Exception e) { error = e; } finally { /* clean-up */ for (Closeable c : new Closeable[] { bis, br, bos }) { if (c != null) { try { c.close(); } catch (Exception e) { /* ignore */ } } } synchronized (stateLock) { running = false; completed = true; } } }
From source file:tufts.vue.URLResource.java
private Properties scrapeHTMLmetaData(URLConnection connection, int maxSearchBytes) throws java.io.IOException { Properties metaData = new Properties(); InputStream byteStream = connection.getInputStream(); if (DEBUG.DND && DEBUG.META) { System.err.println("Getting headers from " + connection); System.err.println("Headers: " + connection.getHeaderFields()); }//from w ww . j av a2s.c o m // note: be sure to call getContentType and don't rely on getting it from the HeaderFields map, // as sometimes it's set by the OS for a file:/// URL when there are no header fields (no http server) // (actually, this is set by java via a mime type table based on file extension, or a guess based on the stream) if (DEBUG.DND) System.err.println("*** getting contentType & encoding..."); final String contentType = connection.getContentType(); final String contentEncoding = connection.getContentEncoding(); final int contentLength = connection.getContentLength(); if (DEBUG.DND) System.err.println("*** contentType [" + contentType + "]"); if (DEBUG.DND) System.err.println("*** contentEncoding [" + contentEncoding + "]"); if (DEBUG.DND) System.err.println("*** contentLength [" + contentLength + "]"); setProperty("url.contentType", contentType); setProperty("url.contentEncoding", contentEncoding); if (contentLength >= 0) setProperty("url.contentLength", contentLength); //if (contentType.toLowerCase().startsWith("text/html") == false) { if (!isHTML()) { // we only currently handle HTML if (DEBUG.Enabled) System.err.println("*** contentType [" + contentType + "] not HTML; skipping title extraction"); return metaData; } if (DEBUG.DND) System.err.println("*** scanning for HTML meta-data..."); try { final BufferedInputStream bufStream = new BufferedInputStream(byteStream, maxSearchBytes); bufStream.mark(maxSearchBytes); final byte[] byteBuffer = new byte[maxSearchBytes]; int bytesRead = 0; int len = 0; // BufferedInputStream still won't read thru a block, so we need to allow // a few reads here to get thru a couple of blocks, so we can get up to // our maxbytes (e.g., a common return chunk count is 1448 bytes, presumably related to the MTU) do { int max = maxSearchBytes - bytesRead; len = bufStream.read(byteBuffer, bytesRead, max); System.out.println("*** read " + len); if (len > 0) bytesRead += len; else if (len < 0) break; } while (len > 0 && bytesRead < maxSearchBytes); if (DEBUG.DND) System.out.println("*** Got total chars: " + bytesRead); String html = new String(byteBuffer, 0, bytesRead); if (DEBUG.DND && DEBUG.META) System.out.println("*** HTML-STRING[" + html + "]"); // first, look for a content encoding, so we can search for and get the title // on a properly encoded character stream String charset = null; Matcher cm = Content_Charset_Regex.matcher(html); if (cm.lookingAt()) { charset = cm.group(1); if (DEBUG.DND) System.err.println("*** found HTML specified charset [" + charset + "]"); setProperty("charset", charset); } if (charset == null && contentEncoding != null) { if (DEBUG.DND || true) System.err.println("*** no charset found: using contentEncoding charset " + contentEncoding); charset = contentEncoding; } final String decodedHTML; if (charset != null) { bufStream.reset(); InputStreamReader decodedStream = new InputStreamReader(bufStream, charset); //InputStreamReader decodedStream = new InputStreamReader(new ByteArrayInputStream(byteBuffer), charset); if (true || DEBUG.DND) System.out.println("*** decoding bytes into characters with official encoding " + decodedStream.getEncoding()); setProperty("contentEncoding", decodedStream.getEncoding()); char[] decoded = new char[bytesRead]; int decodedChars = decodedStream.read(decoded); decodedStream.close(); if (true || DEBUG.DND) System.err.println("*** " + decodedChars + " characters decoded using " + charset); decodedHTML = new String(decoded, 0, decodedChars); } else decodedHTML = html; // we'll just have to go with the default platform charset... // these needed to be left open till the decodedStream was done, which // although it should never need to read beyond what's already buffered, // some internal java code has checks that make sure the underlying stream // isn't closed, even it it isn't used. byteStream.close(); bufStream.close(); Matcher m = HTML_Title_Regex.matcher(decodedHTML); if (m.lookingAt()) { String title = m.group(1); if (true || DEBUG.DND) System.err.println("*** found title [" + title + "]"); metaData.put("title", title.trim()); } } catch (Throwable e) { System.err.println("scrapeHTMLmetaData: " + e); if (DEBUG.DND) e.printStackTrace(); } if (DEBUG.DND || DEBUG.Enabled) System.err.println("*** scrapeHTMLmetaData returning [" + metaData + "]"); return metaData; }
From source file:ch.unifr.pai.twice.widgets.mpproxy.server.JettyProxy.java
public ProcessResult loadFromProxy(HttpServletRequest request, HttpServletResponse response, String uri, String servletPath, String proxyPath) throws ServletException, IOException { //System.out.println("LOAD "+uri); //System.out.println("LOAD "+proxyPath); if ("CONNECT".equalsIgnoreCase(request.getMethod())) { handleConnect(request, response); } else {//from w w w . j a v a2 s .c o m URL url = new URL(uri); URLConnection connection = url.openConnection(); connection.setAllowUserInteraction(false); // Set method HttpURLConnection http = null; if (connection instanceof HttpURLConnection) { http = (HttpURLConnection) connection; http.setRequestMethod(request.getMethod()); http.setInstanceFollowRedirects(false); } // check connection header String connectionHdr = request.getHeader("Connection"); if (connectionHdr != null) { connectionHdr = connectionHdr.toLowerCase(); if (connectionHdr.equals("keep-alive") || connectionHdr.equals("close")) connectionHdr = null; } // copy headers boolean xForwardedFor = false; boolean hasContent = false; Enumeration enm = request.getHeaderNames(); while (enm.hasMoreElements()) { // TODO could be better than this! String hdr = (String) enm.nextElement(); String lhdr = hdr.toLowerCase(); if (_DontProxyHeaders.contains(lhdr)) continue; if (connectionHdr != null && connectionHdr.indexOf(lhdr) >= 0) continue; if ("content-type".equals(lhdr)) hasContent = true; Enumeration vals = request.getHeaders(hdr); while (vals.hasMoreElements()) { String val = (String) vals.nextElement(); if (val != null) { connection.addRequestProperty(hdr, val); xForwardedFor |= "X-Forwarded-For".equalsIgnoreCase(hdr); } } } // Proxy headers connection.setRequestProperty("Via", "1.1 (jetty)"); if (!xForwardedFor) connection.addRequestProperty("X-Forwarded-For", request.getRemoteAddr()); // a little bit of cache control String cache_control = request.getHeader("Cache-Control"); if (cache_control != null && (cache_control.indexOf("no-cache") >= 0 || cache_control.indexOf("no-store") >= 0)) connection.setUseCaches(false); // customize Connection try { connection.setDoInput(true); // do input thang! InputStream in = request.getInputStream(); if (hasContent) { connection.setDoOutput(true); IOUtils.copy(in, connection.getOutputStream()); } // Connect connection.connect(); } catch (Exception e) { e.printStackTrace(); } InputStream proxy_in = null; // handler status codes etc. int code = 500; if (http != null) { proxy_in = http.getErrorStream(); code = http.getResponseCode(); response.setStatus(code, http.getResponseMessage()); } if (proxy_in == null) { try { proxy_in = connection.getInputStream(); } catch (Exception e) { e.printStackTrace(); proxy_in = http.getErrorStream(); } } // clear response defaults. response.setHeader("Date", null); response.setHeader("Server", null); // set response headers int h = 0; String hdr = connection.getHeaderFieldKey(h); String val = connection.getHeaderField(h); while (hdr != null || val != null) { String lhdr = hdr != null ? hdr.toLowerCase() : null; if (hdr != null && val != null && !_DontProxyHeaders.contains(lhdr)) { if (hdr.equalsIgnoreCase("Location")) { val = Rewriter.translateCleanUrl(val, servletPath, proxyPath); } response.addHeader(hdr, val); } h++; hdr = connection.getHeaderFieldKey(h); val = connection.getHeaderField(h); } boolean isGzipped = connection.getContentEncoding() != null && connection.getContentEncoding().contains("gzip"); response.addHeader("Via", "1.1 (jetty)"); // boolean process = connection.getContentType() == null // || connection.getContentType().isEmpty() // || connection.getContentType().contains("html"); boolean process = connection.getContentType() != null && connection.getContentType().contains("text"); if (proxy_in != null) { if (!process) { IOUtils.copy(proxy_in, response.getOutputStream()); proxy_in.close(); } else { InputStream in; if (isGzipped && proxy_in != null && proxy_in.available() > 0) { in = new GZIPInputStream(proxy_in); } else { in = proxy_in; } ByteArrayOutputStream byteArrOS = new ByteArrayOutputStream(); IOUtils.copy(in, byteArrOS); in.close(); if (in != proxy_in) proxy_in.close(); String charset = response.getCharacterEncoding(); if (charset == null || charset.isEmpty()) { charset = "ISO-8859-1"; } String originalContent = new String(byteArrOS.toByteArray(), charset); byteArrOS.close(); return new ProcessResult(originalContent, connection.getContentType(), charset, isGzipped); } } } return null; }
From source file:org.orbeon.oxf.util.Connection.java
/** * Open the connection. This sends request headers, request body, and reads status and response headers. * * @return connection result *//*w w w .j a v a 2 s . c om*/ public ConnectionResult connect() { final boolean isDebugEnabled = indentedLogger.isDebugEnabled(); // Perform connection final String scheme = connectionURL.getProtocol(); if (isHTTPOrHTTPS(scheme) || (httpMethod.equals("GET") && (scheme.equals("file") || scheme.equals("oxf")))) { // http MUST be supported // https SHOULD be supported // file SHOULD be supported try { // Create URL connection object final URLConnection urlConnection = connectionURL.openConnection(); final HTTPURLConnection httpURLConnection = (urlConnection instanceof HTTPURLConnection) ? (HTTPURLConnection) urlConnection : null; // Whether a message body must be sent final boolean hasRequestBody = httpMethod.equals("POST") || httpMethod.equals("PUT"); urlConnection.setDoInput(true); urlConnection.setDoOutput(hasRequestBody); // Configure HTTPURLConnection if (httpURLConnection != null) { // Set state if possible httpURLConnection.setCookieStore(this.cookieStore); // Set method httpURLConnection.setRequestMethod(httpMethod); // Set credentials if (credentials != null) { httpURLConnection.setUsername(credentials.username); if (credentials.password != null) httpURLConnection.setPassword(credentials.password); if (credentials.preemptiveAuthentication != null) httpURLConnection.setPreemptiveAuthentication(credentials.preemptiveAuthentication); if (credentials.domain != null) httpURLConnection.setDomain(credentials.domain); } } // Update request headers { // Handle SOAP // Set request Content-Type, SOAPAction or Accept header if needed final boolean didSOAP = handleSOAP(indentedLogger, httpMethod, headersMap, contentType, hasRequestBody); // Set request content type if (!didSOAP && hasRequestBody) { final String actualContentType = (contentType != null) ? contentType : "application/xml"; headersMap.put("Content-Type", new String[] { actualContentType }); indentedLogger.logDebug(LOG_TYPE, "setting header", "Content-Type", actualContentType); } } // Set headers on connection final List<String> headersToLog; if (headersMap != null && headersMap.size() > 0) { headersToLog = isDebugEnabled ? new ArrayList<String>() : null; for (Map.Entry<String, String[]> currentEntry : headersMap.entrySet()) { final String currentHeaderName = currentEntry.getKey(); final String[] currentHeaderValues = currentEntry.getValue(); if (currentHeaderValues != null) { // Add all header values as "request properties" for (String currentHeaderValue : currentHeaderValues) { urlConnection.addRequestProperty(currentHeaderName, currentHeaderValue); if (headersToLog != null) { headersToLog.add(currentHeaderName); headersToLog.add(currentHeaderValue); } } } } } else { headersToLog = null; } // Log request details except body if (isDebugEnabled) { // Basic connection information final URI connectionURI; try { String userInfo = connectionURL.getUserInfo(); if (userInfo != null) { final int colonIndex = userInfo.indexOf(':'); if (colonIndex != -1) userInfo = userInfo.substring(0, colonIndex + 1) + "xxxxxxxx";// hide password in logs } connectionURI = new URI(connectionURL.getProtocol(), userInfo, connectionURL.getHost(), connectionURL.getPort(), connectionURL.getPath(), connectionURL.getQuery(), connectionURL.getRef()); } catch (URISyntaxException e) { throw new OXFException(e); } indentedLogger.logDebug(LOG_TYPE, "opening URL connection", "method", httpMethod, "URL", connectionURI.toString(), "request Content-Type", contentType); // Log all headers if (headersToLog != null) { final String[] strings = new String[headersToLog.size()]; indentedLogger.logDebug(LOG_TYPE, "request headers", headersToLog.toArray(strings)); } } // Write request body if needed if (hasRequestBody) { // Case of empty body if (messageBody == null) messageBody = new byte[0]; // Log message body for debugging purposes if (logBody) logRequestBody(indentedLogger, contentType, messageBody); // Set request body on connection httpURLConnection.setRequestBody(messageBody); } // Connect urlConnection.connect(); if (httpURLConnection != null) { // Get state if possible // This is either the state we set above before calling connect(), or a new state if we didn't provide any this.cookieStore = httpURLConnection.getCookieStore(); } // Create result final ConnectionResult connectionResult = new ConnectionResult(connectionURL.toExternalForm()) { @Override public void close() { if (getResponseInputStream() != null) { try { getResponseInputStream().close(); } catch (IOException e) { throw new OXFException( "Exception while closing input stream for action: " + connectionURL); } } if (httpURLConnection != null) httpURLConnection.disconnect(); } }; // Get response information that needs to be forwarded { // Status code connectionResult.statusCode = (httpURLConnection != null) ? httpURLConnection.getResponseCode() : 200; // Headers connectionResult.responseHeaders = urlConnection.getHeaderFields(); connectionResult.setLastModified(NetUtils.getLastModifiedAsLong(urlConnection)); // Content-Type connectionResult.setResponseContentType(urlConnection.getContentType(), "application/xml"); } // Log response details except body if (isDebugEnabled) { connectionResult.logResponseDetailsIfNeeded(indentedLogger, Level.DEBUG, LOG_TYPE); } // Response stream connectionResult.setResponseInputStream(urlConnection.getInputStream()); // Log response body if (isDebugEnabled) { connectionResult.logResponseBody(indentedLogger, Level.DEBUG, LOG_TYPE, logBody); } return connectionResult; } catch (IOException e) { throw new ValidationException(e, new LocationData(connectionURL.toExternalForm(), -1, -1)); } } else if (!httpMethod.equals("GET") && (scheme.equals("file") || scheme.equals("oxf"))) { // TODO: implement writing to file: and oxf: // SHOULD be supported (should probably support oxf: as well) throw new OXFException("submission URL scheme not yet implemented: " + scheme); } else if (scheme.equals("mailto")) { // TODO: implement sending mail // MAY be supported throw new OXFException("submission URL scheme not yet implemented: " + scheme); } else { throw new OXFException("submission URL scheme not supported: " + scheme); } }
From source file:org.sakaiproject.lessonbuildertool.tool.beans.SimplePageBean.java
public String getTypeOfUrl(String url) { String mimeType = "text/html"; // try to find the mime type of the remote resource // this is only likely to be a problem if someone is pointing to // a url within Sakai. We think in realistic cases those that are // files will be handled as files, so anything that comes where // will be HTML. That's the default if this fails. URLConnection conn = null;//from w w w .j a v a2 s . c o m try { conn = new URL(new URL(ServerConfigurationService.getServerUrl()), url).openConnection(); conn.setConnectTimeout(10000); conn.setReadTimeout(10000); // generate cookie based on code in RequestFilter.java //String suffix = System.getProperty("sakai.serverId"); //if (suffix == null || suffix.equals("")) // suffix = "sakai"; //Session s = sessionManager.getCurrentSession(); //conn.setRequestProperty("Cookie", "JSESSIONID=" + s.getId() + "." + suffix); conn.connect(); String t = conn.getContentType(); if (t != null && !t.equals("")) { int i = t.indexOf(";"); if (i >= 0) t = t.substring(0, i); t = t.trim(); mimeType = t; } } catch (Exception e) { log.error("getTypeOfUrl connection error " + e); } finally { if (conn != null) { try { conn.getInputStream().close(); } catch (Exception e) { log.error("getTypeOfUrl unable to close " + e); } } } return mimeType; }