Java tutorial
/******************************************************************************* * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University * as Operator of the SLAC National Accelerator Laboratory. * Copyright (c) 2011 Brookhaven National Laboratory. * EPICS archiver appliance is distributed subject to a Software License Agreement found * in file LICENSE that is included with this distribution. *******************************************************************************/ package org.epics.archiverappliance.utils.ui; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.log4j.Logger; import org.epics.archiverappliance.retrieval.mimeresponses.MimeResponse; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONValue; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; /** * Small utility for getting the contents of an URL as various things * @author mshankar * */ public class GetUrlContent { public static final String ARCHAPPL_COMPONENT = "ARCHAPPL_COMPONENT"; private static final Logger logger = Logger.getLogger(GetUrlContent.class); /** * Small utility method for getting the content of an URL as a string * Returns null in case of an exception. * @param urlStr URL * @return URL content */ public static String getURLContent(String urlStr) { try { logger.debug("Getting the contents of " + urlStr + " as a string."); try (InputStream is = getURLContentAsStream(urlStr)) { ByteArrayOutputStream bos = new ByteArrayOutputStream(); IOUtils.copy(is, bos); bos.close(); return new String(bos.toByteArray(), "UTF-8"); } } catch (IOException ex) { logger.error("Exception getting contents of internal URL " + urlStr, ex); } return null; } public static JSONArray getURLContentAsJSONArray(String urlStr) { return getURLContentAsJSONArray(urlStr, true); } /** * Given a URL, get the contents as a JSON Array * @param urlStr URL * @param logErrors If false, do not log any exceptions (they are expected) * @return URL content as JSONArray */ public static JSONArray getURLContentAsJSONArray(String urlStr, boolean logErrors) { try { logger.debug("Getting the contents of " + urlStr + " as a JSON array."); JSONParser parser = new JSONParser(); try (InputStream is = getURLContentAsStream(urlStr)) { return (JSONArray) parser.parse(new InputStreamReader(is)); } } catch (IOException ex) { if (logErrors) { logger.error("Exception getting contents of internal URL " + urlStr, ex); } } catch (ParseException pex) { if (logErrors) { logger.error( "Parse exception getting contents of internal URL " + urlStr + " at " + pex.getPosition(), pex); } } return null; } /** * Given an URL, get the contents as a JSON Object * @param urlStr URL * @return URL content as a JSON Object */ public static JSONObject getURLContentAsJSONObject(String urlStr) { return getURLContentAsJSONObject(urlStr, true); } /** * Given an URL, get the contents as a JSON Object; control logging. * @param urlStr URL * @param logErrors If false, do not log any exceptions (they are expected) * @return URL content as a JSON Object */ public static JSONObject getURLContentAsJSONObject(String urlStr, boolean logErrors) { try { logger.debug("Getting the contents of " + urlStr + " as a JSON object."); JSONParser parser = new JSONParser(); try (InputStream is = getURLContentAsStream(urlStr)) { return (JSONObject) parser.parse(new InputStreamReader(is)); } } catch (IOException ex) { if (logErrors) logger.error("Exception getting contents of internal URL " + urlStr, ex); } catch (ParseException pex) { if (logErrors) logger.error( "Parse exception getting contents of internal URL " + urlStr + " at " + pex.getPosition(), pex); } return null; } /** * Combine JSON arrays from multiple URL's in sequence and return a JSON Array. * We need the supress warnings here as JSONArray is a raw collection. * * @param urlStrs multiple URLs * @return Combined JSON arrays */ @SuppressWarnings("unchecked") public static JSONArray combineJSONArrays(List<String> urlStrs) { JSONArray result = new JSONArray(); for (String urlStr : urlStrs) { try { logger.debug("Getting the contents of " + urlStr + " as a JSON array."); JSONParser parser = new JSONParser(); try (InputStream is = getURLContentAsStream(urlStr)) { JSONArray content = (JSONArray) parser.parse(new InputStreamReader(is)); if (content != null) { result.addAll(content); } else { logger.debug(urlStr + " returned an empty array"); } } } catch (IOException ex) { logger.error("Exception getting contents of internal URL " + urlStr, ex); } catch (ParseException pex) { logger.error( "Parse exception getting contents of internal URL " + urlStr + " at " + pex.getPosition(), pex); } } return result; } /** * Combine JSON arrays of JSON objects from multiple URL's in sequence and sends them to the writer.. * The difference from combineJSONArrays is that inserts a newline after each element. * * @param urlStrs multiple URLs * @param out PrintWriter */ public static void combineJSONArraysAndPrintln(List<String> urlStrs, PrintWriter out) { out.println("["); boolean first = true; for (String urlStr : urlStrs) { try { logger.debug("Getting the contents of " + urlStr + " as a JSON array."); JSONParser parser = new JSONParser(); try (InputStream is = getURLContentAsStream(urlStr)) { JSONArray content = (JSONArray) parser.parse(new InputStreamReader(is)); if (content != null) { for (Object obj : content) { JSONObject jsonObj = (JSONObject) obj; if (first) { first = false; } else { out.println(","); } out.print(JSONValue.toJSONString(jsonObj)); } } else { logger.debug(urlStr + " returned an empty array"); } } } catch (IOException ex) { logger.error("Exception getting contents of internal URL " + urlStr, ex); } catch (ParseException pex) { logger.error( "Parse exception getting contents of internal URL " + urlStr + " at " + pex.getPosition(), pex); } } out.println("]"); } /** * A static utilty method to combine JSON objects * @param dest Details from additionalDetails are added to this. * @param additionalDetails JSONObject */ @SuppressWarnings("unchecked") public static void combineJSONObjects(HashMap<String, String> dest, JSONObject additionalDetails) { if (additionalDetails != null) dest.putAll(additionalDetails); } /** * A static utilty method to combine JSON objects * @param dest Details from additionalDetails are added to this. * @param additionalDetails JSONArray */ @SuppressWarnings("unchecked") public static void combineJSONArrays(LinkedList<Map<String, String>> dest, JSONArray additionalDetails) { if (additionalDetails != null) dest.addAll(additionalDetails); } @SuppressWarnings("unchecked") public static void combineJSONObjectsWithArrays(HashMap<String, Object> dest, JSONObject additionalDetails) { if (additionalDetails != null) dest.putAll(additionalDetails); } /** * Post a JSONArray to a remote server and get the response as a JSON object. * @param url URL * @param array JSONObject Array * @return JSONObject   * @throws IOException   */ public static JSONObject postDataAndGetContentAsJSONObject(String url, LinkedList<JSONObject> array) throws IOException { CloseableHttpClient httpclient = HttpClients.createDefault(); HttpPost postMethod = new HttpPost(url); postMethod.addHeader(ARCHAPPL_COMPONENT, "true"); postMethod.addHeader("Content-Type", MimeTypeConstants.APPLICATION_JSON); postMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ StringEntity archiverValues = new StringEntity(JSONValue.toJSONString(array), ContentType.APPLICATION_JSON); postMethod.setEntity(archiverValues); if (logger.isDebugEnabled()) { logger.debug("About to make a POST with " + url); } HttpResponse response = httpclient.execute(postMethod); HttpEntity entity = response.getEntity(); if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); // ArchiverValuesHandler takes over the burden of closing the input stream. try (InputStream is = entity.getContent()) { JSONObject retval = (JSONObject) JSONValue.parse(new InputStreamReader(is)); return retval; } } else { throw new IOException("HTTP response did not have an entity associated with it"); } } /** * Post a JSONArray to a remote server and get the response as a JSON object. * @param url URL * @param array JSONObject Array * @return JSONArray   * @throws IOException   */ public static JSONArray postDataAndGetContentAsJSONArray(String url, LinkedList<JSONObject> array) throws IOException { CloseableHttpClient httpclient = HttpClients.createDefault(); HttpPost postMethod = new HttpPost(url); postMethod.addHeader(ARCHAPPL_COMPONENT, "true"); postMethod.addHeader("Content-Type", MimeTypeConstants.APPLICATION_JSON); postMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ StringEntity archiverValues = new StringEntity(JSONValue.toJSONString(array), ContentType.APPLICATION_JSON); postMethod.setEntity(archiverValues); if (logger.isDebugEnabled()) { logger.debug("About to make a POST with " + url); } HttpResponse response = httpclient.execute(postMethod); HttpEntity entity = response.getEntity(); if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); // ArchiverValuesHandler takes over the burden of closing the input stream. try (InputStream is = entity.getContent()) { JSONArray retval = (JSONArray) JSONValue.parse(new InputStreamReader(is)); return retval; } } else { throw new IOException("HTTP response did not have an entity associated with it"); } } /** * Post a JSONObject to a remote server and get the response as a JSON object. * @param url URL * @param object A JSONObject * @return JSONObject   * @throws IOException   */ public static JSONObject postObjectAndGetContentAsJSONObject(String url, JSONObject object) throws IOException { CloseableHttpClient httpclient = HttpClients.createDefault(); HttpPost postMethod = new HttpPost(url); postMethod.addHeader(ARCHAPPL_COMPONENT, "true"); postMethod.addHeader("Content-Type", MimeTypeConstants.APPLICATION_JSON); postMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ StringEntity archiverValues = new StringEntity(JSONValue.toJSONString(object), ContentType.APPLICATION_JSON); postMethod.setEntity(archiverValues); if (logger.isDebugEnabled()) { logger.debug("About to make a POST with " + url); } HttpResponse response = httpclient.execute(postMethod); HttpEntity entity = response.getEntity(); if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); // ArchiverValuesHandler takes over the burden of closing the input stream. try (InputStream is = entity.getContent()) { JSONObject retval = (JSONObject) JSONValue.parse(new InputStreamReader(is)); return retval; } } else { throw new IOException("HTTP response did not have an entity associated with it"); } } /** * Post a list of strings to the remove server as a CSV and return the results as a array of JSONObjects * @param url URL * @param paramName   * @param params a list of strings * @return JSONArray   * @throws IOException   */ public static JSONArray postStringListAndGetContentAsJSONArray(String url, String paramName, LinkedList<String> params) throws IOException { StringWriter buf = new StringWriter(); buf.append(paramName); buf.append("="); boolean isFirst = true; for (String param : params) { if (isFirst) { isFirst = false; } else { buf.append(","); } buf.append(param); } CloseableHttpClient httpclient = HttpClients.createDefault(); HttpPost postMethod = new HttpPost(url); postMethod.addHeader("Content-Type", MimeTypeConstants.APPLICATION_FORM_URLENCODED); postMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ StringEntity archiverValues = new StringEntity(buf.toString(), ContentType.APPLICATION_FORM_URLENCODED); postMethod.setEntity(archiverValues); if (logger.isDebugEnabled()) { logger.debug("About to make a POST with " + url); } HttpResponse response = httpclient.execute(postMethod); HttpEntity entity = response.getEntity(); if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); // ArchiverValuesHandler takes over the burden of closing the input stream. try (InputStream is = entity.getContent()) { JSONArray retval = (JSONArray) JSONValue.parse(new InputStreamReader(is)); return retval; } } else { throw new IOException("HTTP response did not have an entity associated with it"); } } /** * Check if we get a valid response from this URL * @param urlStr URL * @return boolean True or False */ public static boolean checkURL(String urlStr) { try { logger.debug("Testing if " + urlStr + " is valid"); try (InputStream is = getURLContentAsStream(urlStr)) { return true; } } catch (IOException ex) { // Ignore any exceptions here as we are only testing if this is a valid URL. } return false; } private static InputStream getURLContentAsStream(String serverURL) throws IOException { CloseableHttpClient httpclient = HttpClients.createDefault(); HttpGet getMethod = new HttpGet(serverURL); getMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ getMethod.addHeader(ARCHAPPL_COMPONENT, "true"); HttpResponse response = httpclient.execute(getMethod); if (response.getStatusLine().getStatusCode() == 200) { HttpEntity entity = response.getEntity(); if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); // ArchiverValuesHandler takes over the burden of closing the input stream. InputStream is = entity.getContent(); return is; } else { throw new IOException("HTTP response did not have an entity associated with it"); } } else { throw new IOException("Invalid status calling " + serverURL + ". Got " + response.getStatusLine().getStatusCode() + response.getStatusLine().getReasonPhrase()); } } /** * Get the contents of a redirect URL and use as reponse for the provided HttpServletResponse. * If possible, pass in error responses as well. * @param redirectURIStr   * @param resp HttpServletResponse * @throws IOException   */ public static void proxyURL(String redirectURIStr, HttpServletResponse resp) throws IOException { CloseableHttpClient httpclient = HttpClients.createDefault(); HttpGet getMethod = new HttpGet(redirectURIStr); getMethod.addHeader("Connection", "close"); // https://www.nuxeo.com/blog/using-httpclient-properly-avoid-closewait-tcp-connections/ try (CloseableHttpResponse response = httpclient.execute(getMethod)) { if (response.getStatusLine().getStatusCode() == 200) { HttpEntity entity = response.getEntity(); HashSet<String> proxiedHeaders = new HashSet<String>(); proxiedHeaders.addAll(Arrays.asList(MimeResponse.PROXIED_HEADERS)); Header[] headers = response.getAllHeaders(); for (Header header : headers) { if (proxiedHeaders.contains(header.getName())) { logger.debug("Adding headerName " + header.getName() + " and value " + header.getValue() + " when proxying request"); resp.addHeader(header.getName(), header.getValue()); } } if (entity != null) { logger.debug("Obtained a HTTP entity of length " + entity.getContentLength()); try (OutputStream os = resp.getOutputStream(); InputStream is = new BufferedInputStream(entity.getContent())) { byte buf[] = new byte[10 * 1024]; int bytesRead = is.read(buf); while (bytesRead > 0) { os.write(buf, 0, bytesRead); resp.flushBuffer(); bytesRead = is.read(buf); } } } else { throw new IOException("HTTP response did not have an entity associated with it"); } } else { logger.error("Invalid status code " + response.getStatusLine().getStatusCode() + " when connecting to URL " + redirectURIStr + ". Sending the errorstream across"); try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { try (InputStream is = new BufferedInputStream(response.getEntity().getContent())) { byte buf[] = new byte[10 * 1024]; int bytesRead = is.read(buf); while (bytesRead > 0) { os.write(buf, 0, bytesRead); bytesRead = is.read(buf); } } resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(response.getStatusLine().getStatusCode(), new String(os.toByteArray())); } } } } }