Java Utililty Methods URL Download

List of utility methods to do URL Download

Description

The list of methods to do URL Download are organized into topic(s).

Method

byte[]fetchURL(String url_string)
Fetch the contents of a URL
try {
    URL url = new URL(url_string);
    URLConnection connection = url.openConnection();
    byte[] bytes = new byte[connection.getContentLength()];
    int offset = 0;
    while (true) {
        int len = connection.getInputStream().read(bytes, offset, bytes.length - offset);
        if (len == -1) {
...
BufferedReaderfetchURL(String urlStr, List errors, Map requestProperties)
requestProperties maps String property to String value; if null, then fetch the content without registering any HTTP request properties.
InputStream urlStream = null;
InputStreamReader urlReader = null;
try {
    URL url = new URL(urlStr);
    URLConnection urlConnection = url.openConnection();
    if (requestProperties != null) {
        for (Map.Entry<String, String> keyValue : requestProperties.entrySet()) {
            urlConnection.setRequestProperty(keyValue.getKey(), keyValue.getValue());
...
intfetchUrl(URL url)
GET the given url.
URLConnection cnx = url.openConnection();
InputStream in = cnx.getInputStream();
byte[] buf = new byte[8192];
int total = 0;
int len = 0;
while ((len = in.read(buf)) >= 0)
    total += len;
return total;
...
byte[]fetchUrl(URL url)
fetch Url
return fetch(url.openStream());
StringfetchURL(URL url)
fetch URL
try {
    URLConnection con = url.openConnection();
    con.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; H010818)");
    BufferedReader reader = new BufferedReader(new InputStreamReader(con.getInputStream()));
    String line;
    StringBuffer pageBuffer = new StringBuffer();
    while ((line = reader.readLine()) != null) {
        pageBuffer.append(line);
...
StringfetchURLContents(String url)
fetch URL Contents
URL u;
try {
    u = new URL(url);
    HttpURLConnection uc = (HttpURLConnection) u.openConnection();
    BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream()));
    String s;
    uc.setConnectTimeout(1000);
    uc.setReadTimeout(1000);
...
Stringget(String url)
get
HttpURLConnection connection = (HttpURLConnection) (new URL(url)).openConnection();
connection.setRequestMethod("GET");
connection.connect();
int code = connection.getResponseCode();
if (code > 300) {
    connection.disconnect();
    return null;
InputStream in = connection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
String line = null;
StringBuilder builder = new StringBuilder();
while ((line = reader.readLine()) != null) {
    builder.append(line).append("\n");
in.close();
reader.close();
connection.disconnect();
return builder.toString();
Stringget(String url)
get
URL obj = new URL(url);
HttpURLConnection con = (HttpURLConnection) obj.openConnection();
con.setRequestMethod("GET");
con.setRequestProperty("User-Agent", USER_AGENT);
int responseCode = con.getResponseCode();
if (responseCode != 200) {
    throw new IOException("Response Code: " + responseCode);
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
try {
    StringBuilder response = new StringBuilder();
    String line;
    while ((line = in.readLine()) != null) {
        response.append(line);
    return response.toString();
} finally {
    in.close();
Stringget(String url)
Run a (blocking) HTTP get request.
String result = "";
try {
    HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
    connection.setInstanceFollowRedirects(true);
    connection.setRequestMethod("GET");
    connection.setRequestProperty("User-Agent", USER_AGENT);
    connection.setConnectTimeout(3000);
    connection.setReadTimeout(10000);
...
JSONObjectget(String url)
GET mechanism
try {
    InputStream response = new URL(url).openStream();
    BufferedReader streamReader = new BufferedReader(new InputStreamReader(response, CHARSET));
    StringBuilder responseStrBuilder = new StringBuilder();
    String inputStr;
    while ((inputStr = streamReader.readLine()) != null) {
        responseStrBuilder.append(inputStr);
    try {
        JSONObject json = new JSONObject(responseStrBuilder.toString());
        return json;
    } catch (JSONException e) {
        System.out.println("Failed reading JSON");
} catch (MalformedURLException e) {
    System.out.println("Invalid URL");
} catch (IOException e) {
    System.out.println("Invalid IO");
return null;