Example usage for java.net HttpURLConnection setRequestProperty

List of usage examples for java.net HttpURLConnection setRequestProperty

  1. HOME
  2. Java
  3. java
  4. java.net.*
  5. HttpURLConnection
  6. setRequestProperty

Introduction

In this page you can find the example usage for java.net HttpURLConnection setRequestProperty.

Prototype

public void setRequestProperty(String key, String value)

Source Link

Document

Sets the general request property.

Usage

From source file:org.alfresco.mobile.android.api.network.NetworkHttpInvoker.java

private static Response invoke(UrlBuilder url, String method, String contentType,
        Map<String, List<String>> httpHeaders, Output writer, boolean forceOutput, BigInteger offset,
        BigInteger length, Map<String, String> params) {
    try {/*from w w w. j av  a  2 s  .  com*/
        // Log.d("URL", url.toString());

        // connect
        HttpURLConnection conn = (HttpURLConnection) (new URL(url.toString())).openConnection();
        conn.setRequestMethod(method);
        conn.setDoInput(true);
        conn.setDoOutput(writer != null || forceOutput);
        conn.setAllowUserInteraction(false);
        conn.setUseCaches(false);
        conn.setRequestProperty("User-Agent", ClientVersion.OPENCMIS_CLIENT);

        // set content type
        if (contentType != null) {
            conn.setRequestProperty("Content-Type", contentType);
        }
        // set other headers
        if (httpHeaders != null) {
            for (Map.Entry<String, List<String>> header : httpHeaders.entrySet()) {
                if (header.getValue() != null) {
                    for (String value : header.getValue()) {
                        conn.addRequestProperty(header.getKey(), value);
                    }
                }
            }
        }

        // range
        BigInteger tmpOffset = offset;
        if ((tmpOffset != null) || (length != null)) {
            StringBuilder sb = new StringBuilder("bytes=");

            if ((tmpOffset == null) || (tmpOffset.signum() == -1)) {
                tmpOffset = BigInteger.ZERO;
            }

            sb.append(tmpOffset.toString());
            sb.append("-");

            if ((length != null) && (length.signum() == 1)) {
                sb.append(tmpOffset.add(length.subtract(BigInteger.ONE)).toString());
            }

            conn.setRequestProperty("Range", sb.toString());
        }

        conn.setRequestProperty("Accept-Encoding", "gzip,deflate");

        // add url form parameters
        if (params != null) {
            DataOutputStream ostream = null;
            OutputStream os = null;
            try {
                os = conn.getOutputStream();
                ostream = new DataOutputStream(os);

                Set<String> parameters = params.keySet();
                StringBuffer buf = new StringBuffer();

                int paramCount = 0;
                for (String it : parameters) {
                    String parameterName = it;
                    String parameterValue = (String) params.get(parameterName);

                    if (parameterValue != null) {
                        parameterValue = URLEncoder.encode(parameterValue, "UTF-8");
                        if (paramCount > 0) {
                            buf.append("&");
                        }
                        buf.append(parameterName);
                        buf.append("=");
                        buf.append(parameterValue);
                        ++paramCount;
                    }
                }
                ostream.writeBytes(buf.toString());
            } finally {
                if (ostream != null) {
                    ostream.flush();
                    ostream.close();
                }
                IOUtils.closeStream(os);
            }
        }

        // send data

        if (writer != null) {
            // conn.setChunkedStreamingMode((64 * 1024) - 1);
            OutputStream connOut = null;
            connOut = conn.getOutputStream();
            OutputStream out = new BufferedOutputStream(connOut, BUFFER_SIZE);
            writer.write(out);
            out.flush();
        }

        // connect
        conn.connect();

        // get stream, if present
        int respCode = conn.getResponseCode();
        InputStream inputStream = null;
        if ((respCode == HttpStatus.SC_OK) || (respCode == HttpStatus.SC_CREATED)
                || (respCode == HttpStatus.SC_NON_AUTHORITATIVE_INFORMATION)
                || (respCode == HttpStatus.SC_PARTIAL_CONTENT)) {
            inputStream = conn.getInputStream();
        }

        // get the response
        return new Response(respCode, conn.getResponseMessage(), conn.getHeaderFields(), inputStream,
                conn.getErrorStream());
    } catch (Exception e) {
        throw new CmisConnectionException("Cannot access " + url + ": " + e.getMessage(), e);
    }
}

From source file:gribbit.util.RequestBuilder.java

/**
 * Make a GET or POST request, handling up to 6 redirects, and return the response. If isBinaryResponse is true,
 * returns a byte[] array, otherwise returns the response as a String.
 *//*  w  ww .java2  s  .  c  om*/
private static Object makeRequest(String url, String[] keyValuePairs, boolean isGET, boolean isBinaryResponse,
        int redirDepth) {
    if (redirDepth > 6) {
        throw new IllegalArgumentException("Too many redirects");
    }
    HttpURLConnection connection = null;
    try {
        // Add the URL query params if this is a GET request
        String reqURL = isGET ? url + "?" + WebUtils.buildQueryString(keyValuePairs) : url;
        connection = (HttpURLConnection) new URL(reqURL).openConnection();
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.setInstanceFollowRedirects(false);
        connection.setRequestMethod(isGET ? "GET" : "POST");
        connection.setUseCaches(false);
        connection.setRequestProperty("User-Agent",
                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
                        + "Chrome/43.0.2357.125 Safari/537.36");
        if (!isGET) {
            // Send the body if this is a POST request
            connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
            connection.setRequestProperty("charset", "utf-8");
            String params = WebUtils.buildQueryString(keyValuePairs);
            connection.setRequestProperty("Content-Length", Integer.toString(params.length()));
            try (DataOutputStream w = new DataOutputStream(connection.getOutputStream())) {
                w.writeBytes(params);
                w.flush();
            }
        }
        if (connection.getResponseCode() == HttpResponseStatus.FOUND.code()) {
            // Follow a redirect. For safety, the params are not passed on, and the method is forced to GET.
            return makeRequest(connection.getHeaderField("Location"), /* keyValuePairs = */null, /* isGET = */
                    true, isBinaryResponse, redirDepth + 1);
        } else if (connection.getResponseCode() == HttpResponseStatus.OK.code()) {
            // For 200 OK, return the text of the response
            if (isBinaryResponse) {
                ByteArrayOutputStream output = new ByteArrayOutputStream(32768);
                IOUtils.copy(connection.getInputStream(), output);
                return output.toByteArray();
            } else {
                StringWriter writer = new StringWriter(1024);
                IOUtils.copy(connection.getInputStream(), writer, "UTF-8");
                return writer.toString();
            }
        } else {
            throw new IllegalArgumentException(
                    "Got non-OK HTTP response code: " + connection.getResponseCode());
        }
    } catch (Exception e) {
        throw new IllegalArgumentException(
                "Exception during " + (isGET ? "GET" : "POST") + " request: " + e.getMessage(), e);
    } finally {
        if (connection != null) {
            try {
                connection.disconnect();
            } catch (Exception e) {
            }
        }
    }
}

From source file:crow.weibo.util.WeiboUtil.java

public static String multipartPost(HttpURLConnection conn, List<PostParameter> params) throws IOException {
    OutputStream os;/* ww w  .ja va  2 s  .c o  m*/
    List<PostParameter> dataparams = new ArrayList<PostParameter>();
    for (PostParameter key : params) {
        if (key.isFile()) {
            dataparams.add(key);
        }
    }

    String BOUNDARY = Util.md5(String.valueOf(System.currentTimeMillis()));

    conn.setDoOutput(true);
    conn.setDoInput(true);
    conn.setUseCaches(false);
    conn.setRequestMethod("POST");
    conn.setRequestProperty("Connection", "Keep-Alive");
    conn.setRequestProperty("Charsert", "UTF-8");

    conn.setRequestProperty("Content-Type", "multipart/form-data;boundary=" + BOUNDARY);

    ByteArrayBuffer buff = new ByteArrayBuffer(1000);

    for (PostParameter p : params) {
        byte[] arr = p.toMultipartByte(BOUNDARY, "UTF-8");
        buff.append(arr, 0, arr.length);
    }
    String end = "--" + BOUNDARY + "--" + "\r\n";
    byte[] endArr = end.getBytes();
    buff.append(endArr, 0, endArr.length);

    conn.setRequestProperty("Content-Length", buff.length() + "");
    conn.connect();
    os = new BufferedOutputStream(conn.getOutputStream());
    os.write(buff.toByteArray());
    buff.clear();
    os.flush();
    String response = "";
    response = Util.inputStreamToString(conn.getInputStream());
    return response;
}

From source file:com.fota.Link.sdpApi.java

public static String getCtnInfo(String ncn) throws JDOMException {
    String resultStr = "";
    String logData = "";
    try {//from  w ww . j  av a 2s .  c  om
        String cpId = PropUtil.getPropValue("sdp3g.id");
        String cpPw = PropUtil.getPropValue("sdp3g.pw");
        String authorization = cpId + ":" + cpPw;
        logData = "\r\n---------- Get Ctn Req Info start ----------\r\n";
        logData += " getCtnRequestInfo - authorization : " + authorization;
        byte[] encoded = Base64.encodeBase64(authorization.getBytes());
        authorization = new String(encoded);

        String contractType = "0";
        String contractNum = ncn.substring(0, 9);
        String customerId = ncn.substring(10, ncn.length() - 1);

        JSONObject reqJObj = new JSONObject();
        reqJObj.put("transactionid", "");
        reqJObj.put("sequenceno", "");
        reqJObj.put("userid", "");
        reqJObj.put("screenid", "");
        reqJObj.put("CONTRACT_NUM", contractNum);
        reqJObj.put("CUSTOMER_ID", customerId);
        reqJObj.put("CONTRACT_TYPE", contractType);

        authorization = "Basic " + authorization;

        String endPointUrl = PropUtil.getPropValue("sdp.oif555.url");

        logData += "\r\n getCtnRequestInfo - endPointUrl : " + endPointUrl;
        logData += "\r\n getCtnRequestInfo - authorization(encode) : " + authorization;
        logData += "\r\n getCtnRequestInfo - Content-type : application/json";
        logData += "\r\n getCtnRequestInfo - RequestMethod : POST";
        logData += "\r\n getCtnRequestInfo - json : " + reqJObj.toString();
        logData += "\r\n---------- Get Ctn Req Info end ----------";

        logger.info(logData);

        // connection
        URL url = new URL(endPointUrl);
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();

        connection.setRequestProperty("Authorization", authorization);
        connection.setRequestProperty("Content-type", "application/json");

        connection.setRequestMethod("POST");
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.connect();

        // output
        DataOutputStream wr = new DataOutputStream(connection.getOutputStream());
        wr.writeBytes(reqJObj.toJSONString());
        wr.flush();
        wr.close();

        // input
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String inputLine = "";
        StringBuffer response = new StringBuffer();

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }

        in.close();

        JSONParser jsonParser = new JSONParser();
        JSONObject respJsonObject = (JSONObject) jsonParser.parse(response.toString());

        //         String respTransactionId = (String) respJsonObject.get("transactionid");
        //         String respSequenceNo = (String) respJsonObject.get("sequenceno");
        //         String respReturnCode = (String) respJsonObject.get("returncode");
        //         String respReturnDescription = (String) respJsonObject.get("returndescription");
        //         String respErrorCode = (String) respJsonObject.get("errorcode");
        //         String respErrorDescription = (String) respJsonObject.get("errordescription");
        String respCtn = (String) respJsonObject.get("ctn");
        //         String respSubStatus = (String) respJsonObject.get("sub_status");
        //         String respSubStatusDate = (String) respJsonObject.get("sub_status_date");

        resultStr = respCtn;

        //resultStr = resValue;         
        //         resultStr = java.net.URLDecoder.decode(resultStr, "euc-kr");
        connection.disconnect();

    } catch (Exception e) {
        e.printStackTrace();
    }

    return resultStr;
}

From source file:edu.jhu.cvrg.waveform.utility.WebServiceUtility.java

public static Map<String, String> annotationJSONLookup(String restURL, String... key) {

    Map<String, String> ret = null;
    URL url;/*from   www  .j av  a  2 s.co  m*/
    HttpURLConnection conn = null;
    try {
        if (System.currentTimeMillis() > waitUntil) {
            url = new URL(restURL);

            conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setRequestProperty("Accept", "application/json");

            BufferedReader in = new BufferedReader(new InputStreamReader((conn.getInputStream())));

            String jsonSrc = in.readLine();
            in.close();

            JSONObject jsonObject = new JSONObject(jsonSrc);
            ret = new HashMap<String, String>();

            for (int i = 0; i < key.length; i++) {
                Object atr = jsonObject.get(key[i]);
                String value = "";
                if (atr instanceof JSONArray) {
                    JSONArray array = ((JSONArray) atr);
                    for (int j = 0; j < array.length(); j++) {
                        value += array.getString(j);
                    }
                } else {
                    value = atr.toString();
                }
                if (value.isEmpty()) {
                    value = "No " + key[i] + " found";
                }
                ret.put(key[i], value);
            }
        } else {
            log.warn("Waiting the bioportal server");
        }

    } catch (MalformedURLException mue) {
        mue.printStackTrace();
    } catch (IOException ioe) {
        //wait for 1 minute
        waitUntil = System.currentTimeMillis() + (1000 * 60);

        log.error(ioe.getMessage());
    } catch (JSONException e) {
        e.printStackTrace();
    }
    return ret;
}

From source file:eu.geopaparazzi.library.network.NetworkUtilities.java

/**
 * Sends an HTTP GET request to a url/*from w  w w  . ja v  a  2  s .c  o  m*/
 *
 * @param urlStr - The URL of the server. (Example: " http://www.yahoo.com/search")
 * @param file the output file. If it is a folder, it tries to get the file name from the header.
 * @param requestParameters - all the request parameters (Example: "param1=val1&param2=val2"). 
 *           Note: This method will add the question mark (?) to the request - 
 *           DO NOT add it yourself
 * @param user
 * @param password
 * @return the file written. 
 * @throws Exception 
 */
public static File sendGetRequest4File(String urlStr, File file, String requestParameters, String user,
        String password) throws Exception {
    if (requestParameters != null && requestParameters.length() > 0) {
        urlStr += "?" + requestParameters;
    }
    HttpURLConnection conn = makeNewConnection(urlStr);
    conn.setRequestMethod("GET");
    // conn.setDoOutput(true);
    conn.setDoInput(true);
    // conn.setChunkedStreamingMode(0);
    conn.setUseCaches(false);

    if (user != null && password != null) {
        conn.setRequestProperty("Authorization", getB64Auth(user, password));
    }
    conn.connect();

    if (file.isDirectory()) {
        // try to get the header
        String headerField = conn.getHeaderField("Content-Disposition");
        String fileName = null;
        if (headerField != null) {
            String[] split = headerField.split(";");
            for (String string : split) {
                String pattern = "filename=";
                if (string.toLowerCase().startsWith(pattern)) {
                    fileName = string.replaceFirst(pattern, "");
                    break;
                }
            }
        }
        if (fileName == null) {
            // give a name
            fileName = "FILE_" + LibraryConstants.TIMESTAMPFORMATTER.format(new Date());
        }
        file = new File(file, fileName);
    }

    InputStream in = null;
    FileOutputStream out = null;
    try {
        in = conn.getInputStream();
        out = new FileOutputStream(file);

        byte[] buffer = new byte[(int) maxBufferSize];
        int bytesRead = in.read(buffer, 0, (int) maxBufferSize);
        while (bytesRead > 0) {
            out.write(buffer, 0, bytesRead);
            bytesRead = in.read(buffer, 0, (int) maxBufferSize);
        }
        out.flush();

    } finally {
        if (in != null)
            in.close();
        if (out != null)
            out.close();
    }
    return file;
}

From source file:com.gmobi.poponews.util.HttpHelper.java

public static Response upload(String url, InputStream is) {
    Response response = new Response();
    String boundary = Long.toHexString(System.currentTimeMillis());
    HttpURLConnection connection = null;
    try {//from w w  w  . j  a v a  2  s  .co  m
        URL httpURL = new URL(url);
        connection = (HttpURLConnection) httpURL.openConnection();
        connection.setConnectTimeout(15000);
        connection.setReadTimeout(30000);
        connection.setRequestMethod("POST");
        connection.setDoOutput(true);
        connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary);
        byte[] st = ("--" + boundary + "\r\n"
                + "Content-Disposition: form-data; name=\"file\"; filename=\"data\"\r\n"
                + "Content-Type: application/octet-stream; charset=UTF-8\r\n"
                + "Content-Transfer-Encoding: binary\r\n\r\n").getBytes();
        byte[] en = ("\r\n--" + boundary + "--\r\n").getBytes();
        connection.setRequestProperty("Content-Length", String.valueOf(st.length + en.length + is.available()));
        OutputStream os = connection.getOutputStream();
        os.write(st);
        FileHelper.copy(is, os);
        os.write(en);
        os.flush();
        os.close();
        response.setStatusCode(connection.getResponseCode());
        connection = null;
    } catch (Exception e) {
        Logger.error(e);
    }

    return response;
}

From source file:org.apache.mycat.advisor.common.net.http.HttpService.java

/**
 * ? header?/* w  ww. j a  v  a2s .  c o  m*/
 *
 * @param requestUrl
 * @param requestMethod
 * @param WithTokenHeader
 * @param token
 * @return
 */
public static String doHttpRequest(String requestUrl, String requestMethod, Boolean WithTokenHeader,
        String token) {
    String result = null;
    InetAddress ipaddr;
    int responseCode = -1;
    try {
        ipaddr = InetAddress.getLocalHost();
        StringBuffer buffer = new StringBuffer();
        URL url = new URL(requestUrl);
        HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
        httpUrlConn.setDoOutput(true);
        httpUrlConn.setDoInput(true);
        httpUrlConn.setUseCaches(false);

        httpUrlConn.setUseCaches(false);
        httpUrlConn.setRequestProperty("Accept-Charset", DEFAULT_CHARSET);
        httpUrlConn.setRequestProperty("Content-Type", "application/json;charset=" + DEFAULT_CHARSET);
        if (WithTokenHeader) {
            if (token == null) {
                throw new IllegalStateException("Oauth2 token is not set!");
            }
            httpUrlConn.setRequestProperty("Authorization", "OAuth2 " + token);
            httpUrlConn.setRequestProperty("API-RemoteIP", ipaddr.getHostAddress());
        }
        // ?GET/POST
        httpUrlConn.setRequestMethod(requestMethod);

        if ("GET".equalsIgnoreCase(requestMethod))
            httpUrlConn.connect();

        //            // ????
        //            if (null != outputJson) {
        //                OutputStream outputStream = httpUrlConn.getOutputStream();
        //                //??
        //                outputStream.write(outputJson.getBytes(DEFAULT_CHARSET));
        //                outputStream.close();
        //            }

        // ???
        InputStream inputStream = httpUrlConn.getInputStream();
        InputStreamReader inputStreamReader = new InputStreamReader(inputStream, DEFAULT_CHARSET);
        BufferedReader bufferedReader = new BufferedReader(inputStreamReader);

        String str = null;
        while ((str = bufferedReader.readLine()) != null) {
            buffer.append(str);
        }
        result = buffer.toString();
        bufferedReader.close();
        inputStreamReader.close();
        // ?
        inputStream.close();
        httpUrlConn.disconnect();
    } catch (ConnectException ce) {
        logger.error("server connection timed out.", ce);
    } catch (Exception e) {
        logger.error("http request error:", e);
    } finally {
        return result;
    }
}

From source file:com.fota.Link.sdpApi.java

public static String getNcnInfo(String CTN) throws JDOMException {
    String resultStr = "";
    String logData = "";
    try {//from   ww  w .ja  v  a  2  s.  co m
        String endPointUrl = PropUtil.getPropValue("sdp.oif516.url");

        String strRequest = "<soapenv:Envelope xmlns:soapenv='http://schemas.xmlsoap.org/soap/envelope/' "
                + "xmlns:oas='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd'"
                + " xmlns:sdp='http://kt.com/sdp'>" + "     <soapenv:Header>" + "         <oas:Security>"
                + "             <oas:UsernameToken>" + "                 <oas:Username>"
                + PropUtil.getPropValue("sdp.id") + "</oas:Username>" + "                 <oas:Password>"
                + PropUtil.getPropValue("sdp.pw") + "</oas:Password>" + "             </oas:UsernameToken>"
                + "         </oas:Security>" + "     </soapenv:Header>"

                + "     <soapenv:Body>" + "         <sdp:getBasicUserInfoAndMarketInfoRequest>"
                + "         <!--You may enterthe following 6 items in any order-->"
                + "             <sdp:CALL_CTN>" + CTN + "</sdp:CALL_CTN>"
                + "         </sdp:getBasicUserInfoAndMarketInfoRequest>\n" + "     </soapenv:Body>\n"
                + "</soapenv:Envelope>";

        logData = "\r\n---------- Get Ncn Req Info start ----------\r\n";
        logData += " get Ncn Req Info - endPointUrl : " + endPointUrl;
        logData += "\r\n get Ncn Req Info - Content-type : text/xml;charset=utf-8";
        logData += "\r\n get Ncn Req Info - RequestMethod : POST";
        logData += "\r\n get Ncn Req Info - xml : " + strRequest;
        logData += "\r\n---------- Get Ncn Req Info end ----------";

        logger.info(logData);

        // connection
        URL url = new URL(endPointUrl);
        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
        connection.setRequestProperty("Content-type", "text/xml;charset=utf-8");
        connection.setRequestMethod("POST");
        connection.setDoOutput(true);
        connection.setDoInput(true);
        connection.connect();

        // output
        OutputStream os = connection.getOutputStream();
        // os.write(strRequest.getBytes(), 0, strRequest.length());
        os.write(strRequest.getBytes("utf-8"));
        os.flush();
        os.close();

        // input
        InputStream is = connection.getInputStream();
        BufferedReader br = new BufferedReader(new InputStreamReader(is, "utf-8"));
        String line = "";
        String resValue = "";
        String parseStr = "";
        while ((line = br.readLine()) != null) {
            System.out.println(line);
            parseStr = line;

        }

        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = factory.newDocumentBuilder();
        InputSource temp = new InputSource();
        temp.setCharacterStream(new StringReader(parseStr));
        Document doc = builder.parse(temp); //xml?

        NodeList list = doc.getElementsByTagName("*");

        int i = 0;
        Element element;
        String contents;

        String contractNum = "";
        String customerId = "";
        while (list.item(i) != null) {
            element = (Element) list.item(i);
            if (element.hasChildNodes()) {
                contents = element.getFirstChild().getNodeValue();
                System.out.println(element.getNodeName() + " / " + element.getFirstChild().getNodeName());

                if (element.getNodeName().equals("sdp:NS_CONTRACT_NUM")) {
                    //                  resultStr = element.getFirstChild().getNodeValue();
                    contractNum = element.getFirstChild().getNodeValue();
                }
                if (element.getNodeName().equals("sdp:NS_CUSTOMER_ID")) {
                    customerId = element.getFirstChild().getNodeValue();
                }

                //               System.out.println(" >>>>> " + contents);
            }
            i++;
        }

        //         System.out.println("contractNum : " + contractNum + " / cusomerId : " + customerId);

        resultStr = getNcnFromContractnumAndCustomerId(contractNum, customerId);
        //         System.out.println("ncn : " + resultStr);

        //resultStr = resValue;         
        //         resultStr = java.net.URLDecoder.decode(resultStr, "euc-kr");
        connection.disconnect();

    } catch (Exception e) {
        e.printStackTrace();
    }

    return resultStr;
}

From source file:edu.stanford.muse.index.NER.java

/** returns a list of <name, #occurrences> */
public static List<Pair<String, Float>> namesFromURL(String url, boolean removeCommonNames)
        throws ClassCastException, IOException, ClassNotFoundException {
    // only http conns allowed currently
    Indexer.log.info(url);/*from  ww  w .jav  a2  s. c o  m*/
    HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
    conn.setInstanceFollowRedirects(true);
    conn.setRequestProperty("User-agent",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
    conn.connect();
    Indexer.log.info("url for extracting names:" + conn.getURL());
    byte[] b = Util.getBytesFromStream(conn.getInputStream());
    String text = new String(b, "UTF-8");
    text = Util.unescapeHTML(text);
    org.jsoup.nodes.Document doc = Jsoup.parse(text);
    text = doc.body().text();
    return namesFromText(text, removeCommonNames);
}