Example usage for java.net HttpURLConnection getHeaderField

List of usage examples for java.net HttpURLConnection getHeaderField

  1. HOME
  2. Java
  3. java
  4. java.net.*
  5. HttpURLConnection
  6. getHeaderField

Introduction

In this page you can find the example usage for java.net HttpURLConnection getHeaderField.

Prototype

public String getHeaderField(int n)

Source Link

Document

Returns the value for the n th header field.

Usage

From source file:fr.zcraft.zbanque.network.PacketSender.java

private static HTTPResponse makeRequest(String url, PacketPlayOut.PacketType method, String data)
        throws Throwable {
    // ***  REQUEST  ***

    final URL urlObj = new URL(url);
    final HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection();

    connection.setRequestMethod(method.name());
    connection.setRequestProperty("User-Agent", USER_AGENT);

    authenticateRequest(connection);/*from   ww  w  . jav a 2  s.co m*/

    connection.setDoOutput(true);

    try {
        try {
            connection.connect();
        } catch (IOException ignored) {
        }

        if (method == PacketPlayOut.PacketType.POST) {
            DataOutputStream out = null;
            try {
                out = new DataOutputStream(connection.getOutputStream());
                if (data != null)
                    out.writeBytes(data);
                out.flush();
            } finally {
                if (out != null)
                    out.close();
            }
        }

        // ***  RESPONSE  ***

        int responseCode;
        boolean failed = false;

        try {
            responseCode = connection.getResponseCode();
        } catch (IOException e) {
            // HttpUrlConnection will throw an IOException if any 4XX
            // response is sent. If we request the status again, this
            // time the internal status will be properly set, and we'll be
            // able to retrieve it.
            // Thanks to Iigo.
            responseCode = connection.getResponseCode();
            failed = true;
        }

        BufferedReader in = null;
        String body = "";
        try {
            InputStream stream;
            try {
                stream = connection.getInputStream();
            } catch (IOException e) {
                // Same as before
                stream = connection.getErrorStream();
                failed = true;
            }

            in = new BufferedReader(new InputStreamReader(stream));
            StringBuilder responseBuilder = new StringBuilder();

            String inputLine;
            while ((inputLine = in.readLine()) != null) {
                responseBuilder.append(inputLine);
            }

            body = responseBuilder.toString();
        } finally {
            if (in != null)
                in.close();
        }

        HTTPResponse response = new HTTPResponse();
        response.setResponseCode(responseCode, failed);
        response.setResponseBody(body);

        int i = 0;
        String headerName, headerContent;
        while ((headerName = connection.getHeaderFieldKey(i)) != null) {
            headerContent = connection.getHeaderField(i);
            response.addHeader(headerName, headerContent);
        }

        // ***  REDIRECTION  ***

        switch (responseCode) {
        case 301:
        case 302:
        case 307:
        case 308:
            if (response.getHeaders().containsKey("Location")) {
                response = makeRequest(response.getHeaders().get("Location"), method, data);
            }
        }

        // ***  END  ***

        return response;
    } finally {
        connection.disconnect();
    }
}

From source file:fr.gael.dhus.olingo.ODataClient.java

/**
 * Performs the execution of an OData command through HTTP.
 * //from   www  . j  ava 2s  . c  o  m
 * @param absolute_uri The not that relative URI to query.
 * @param content_type The content type can be JSON, XML, Atom+XML,
 *    see {@link OdataContentType}.
 * @param http_method {@code "POST", "GET", "PUT", "DELETE", ...}
 * 
 * @return The response as a stream. You may assume it's UTF-8 encoded.
 * 
 * @throws HttpException if the server emits an HTTP error code.
 * @throws IOException if an error occurred connecting to the server.
 */
private InputStream execute(String absolute_uri, ContentType content_type, String http_method)
        throws IOException {
    URL url = new URL(absolute_uri);
    HttpURLConnection connection = (HttpURLConnection) url.openConnection();

    // HTTP Basic Authentication.
    String userpass = this.username + ":" + this.password;
    String basicAuth = "Basic " + new String(new Base64().encode(userpass.getBytes()));
    connection.setRequestProperty("Authorization", basicAuth);

    // GET, POST, ...
    connection.setRequestMethod(http_method);

    // `Accept` for GET, `Content-Type` for POST and PUT.
    connection.setRequestProperty("Accept", content_type.type());

    connection.connect();

    int resp_code = connection.getResponseCode();
    // 2XX == success, 3XX == redirect (handled by the HTTPUrlConnection)
    if (resp_code == 200) {
        InputStream content = connection.getInputStream();

        content = logRawContent(http_method + " request on uri '" + absolute_uri + "' with content:\n", content,
                "\n");

        return content;
    } else if (resp_code >= 300 && resp_code < 400) {
        // HttpURLConnection should follow redirections automatically,
        // but won't follow if the protocol changes.
        // See https://bugs.openjdk.java.net/browse/JDK-4620571
        // If the scheme has changed (http -> https) follow the redirection.

        String redi_uri = connection.getHeaderField("Location");

        if (redi_uri == null || redi_uri.isEmpty())
            throw new HttpException(connection.getResponseCode(),
                    connection.getResponseMessage() + " redirection failure.");

        if (!redi_uri.startsWith("https"))
            throw new HttpException(connection.getResponseCode(),
                    connection.getResponseMessage() + " unsecure redirection.");

        LOGGER.debug("Attempting redirection to " + redi_uri);
        connection.disconnect();

        return execute(redi_uri, content_type, http_method);
    } else {
        throw new HttpException(connection.getResponseCode(), connection.getResponseMessage());
    }
}

From source file:org.drools.guvnor.server.jaxrs.BasicPackageResourceTest.java

@Test
@RunAsClient/*from  w w w  .j a  va  2 s.  c o m*/
public void testGetPackageSource(@ArquillianResource URL baseURL) throws Exception {
    URL url = new URL(baseURL, "rest/packages/restPackage1/source");
    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
    connection.setRequestProperty("Authorization",
            "Basic " + new Base64().encodeToString(("admin:admin".getBytes())));
    connection.setRequestMethod("GET");
    connection.setRequestProperty("Accept", MediaType.WILDCARD);
    connection.connect();

    assertEquals(200, connection.getResponseCode());
    assertEquals(MediaType.TEXT_PLAIN, connection.getContentType());
    String result = IOUtils.toString(connection.getInputStream());

    assertEquals("attachment; filename=restPackage1", connection.getHeaderField("Content-Disposition"));
    assertTrue(result.indexOf("package restPackage1") >= 0);
    assertTrue(result.indexOf("import com.billasurf.Board") >= 0);
    assertTrue(result.indexOf("global com.billasurf.Person customer2") >= 0);
    assertTrue(result.indexOf("function void foo() { System.out.println(version 2); }") >= 0);
    assertTrue(result.indexOf("declare Album2") >= 0);
}

From source file:org.apache.tez.engine.common.shuffle.impl.Fetcher.java

/**
 * The crux of the matter.../*  w w w .  ja va 2 s .  c om*/
 * 
 * @param host {@link MapHost} from which we need to  
 *              shuffle available map-outputs.
 */
@VisibleForTesting
protected void copyFromHost(MapHost host) throws IOException {
    // Get completed maps on 'host'
    List<TezTaskAttemptID> maps = scheduler.getMapsForHost(host);

    // Sanity check to catch hosts with only 'OBSOLETE' maps, 
    // especially at the tail of large jobs
    if (maps.size() == 0) {
        return;
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + maps);
    }

    // List of maps to be fetched yet
    Set<TezTaskAttemptID> remaining = new HashSet<TezTaskAttemptID>(maps);

    // Construct the url and connect
    DataInputStream input;
    boolean connectSucceeded = false;

    try {
        URL url = getMapOutputURL(host, maps);
        HttpURLConnection connection = openConnection(url);

        // generate hash of the url
        String msgToEncode = SecureShuffleUtils.buildMsgFrom(url);
        String encHash = SecureShuffleUtils.hashFromString(msgToEncode, jobTokenSecret);

        // put url hash into http header
        connection.addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
        // set the read timeout
        connection.setReadTimeout(readTimeout);
        // put shuffle version into http header
        connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
        connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
                ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
        connect(connection, connectionTimeout);
        connectSucceeded = true;
        input = new DataInputStream(connection.getInputStream());

        // Validate response code
        int rc = connection.getResponseCode();
        if (rc != HttpURLConnection.HTTP_OK) {
            throw new IOException("Got invalid response code " + rc + " from " + url + ": "
                    + connection.getResponseMessage());
        }
        // get the shuffle version
        if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME
                .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME))
                || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION
                        .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION))) {
            throw new IOException("Incompatible shuffle response version");
        }
        // get the replyHash which is HMac of the encHash we sent to the server
        String replyHash = connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH);
        if (replyHash == null) {
            throw new IOException("security validation of TT Map output failed");
        }
        LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash=" + replyHash);
        // verify that replyHash is HMac of encHash
        SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret);
        LOG.info("for url=" + msgToEncode + " sent hash and receievd reply");
    } catch (IOException ie) {
        ioErrs.increment(1);
        LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie);

        // If connect did not succeed, just mark all the maps as failed,
        // indirectly penalizing the host
        if (!connectSucceeded) {
            for (TezTaskAttemptID left : remaining) {
                scheduler.copyFailed(left, host, connectSucceeded);
            }
        } else {
            // If we got a read error at this stage, it implies there was a problem
            // with the first map, typically lost map. So, penalize only that map
            // and add the rest
            TezTaskAttemptID firstMap = maps.get(0);
            scheduler.copyFailed(firstMap, host, connectSucceeded);
        }

        // Add back all the remaining maps, WITHOUT marking them as failed
        for (TezTaskAttemptID left : remaining) {
            scheduler.putBackKnownMapOutput(host, left);
        }

        return;
    }

    try {
        // Loop through available map-outputs and fetch them
        // On any error, faildTasks is not null and we exit
        // after putting back the remaining maps to the 
        // yet_to_be_fetched list and marking the failed tasks.
        TezTaskAttemptID[] failedTasks = null;
        while (!remaining.isEmpty() && failedTasks == null) {
            failedTasks = copyMapOutput(host, input, remaining);
        }

        if (failedTasks != null && failedTasks.length > 0) {
            LOG.warn("copyMapOutput failed for tasks " + Arrays.toString(failedTasks));
            for (TezTaskAttemptID left : failedTasks) {
                scheduler.copyFailed(left, host, true);
            }
        }

        IOUtils.cleanup(LOG, input);

        // Sanity check
        if (failedTasks == null && !remaining.isEmpty()) {
            throw new IOException(
                    "server didn't return all expected map outputs: " + remaining.size() + " left.");
        }
    } finally {
        for (TezTaskAttemptID left : remaining) {
            scheduler.putBackKnownMapOutput(host, left);
        }
    }
}

From source file:org.apache.tez.runtime.library.common.shuffle.impl.Fetcher.java

/**
 * The crux of the matter.../*w w w  . j a  v a  2  s  . com*/
 * 
 * @param host {@link MapHost} from which we need to  
 *              shuffle available map-outputs.
 */
@VisibleForTesting
protected void copyFromHost(MapHost host) throws IOException {
    // Get completed maps on 'host'
    List<InputAttemptIdentifier> srcAttempts = scheduler.getMapsForHost(host);

    // Sanity check to catch hosts with only 'OBSOLETE' maps, 
    // especially at the tail of large jobs
    if (srcAttempts.size() == 0) {
        return;
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("Fetcher " + id + " going to fetch from " + host + " for: " + srcAttempts);
    }

    // List of maps to be fetched yet
    remaining = new LinkedHashSet<InputAttemptIdentifier>(srcAttempts);

    // Construct the url and connect
    DataInputStream input;
    boolean connectSucceeded = false;

    try {
        URL url = getMapOutputURL(host, srcAttempts);
        HttpURLConnection connection = openConnection(url);

        // generate hash of the url
        String msgToEncode = SecureShuffleUtils.buildMsgFrom(url);
        String encHash = SecureShuffleUtils.hashFromString(msgToEncode, jobTokenSecret);

        // put url hash into http header
        connection.addRequestProperty(SecureShuffleUtils.HTTP_HEADER_URL_HASH, encHash);
        // set the read timeout
        connection.setReadTimeout(readTimeout);
        // put shuffle version into http header
        connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_NAME, ShuffleHeader.DEFAULT_HTTP_HEADER_NAME);
        connection.addRequestProperty(ShuffleHeader.HTTP_HEADER_VERSION,
                ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION);
        connect(connection, connectionTimeout);
        connectSucceeded = true;
        input = new DataInputStream(connection.getInputStream());

        // Validate response code
        int rc = connection.getResponseCode();
        if (rc != HttpURLConnection.HTTP_OK) {
            throw new IOException("Got invalid response code " + rc + " from " + url + ": "
                    + connection.getResponseMessage());
        }
        // get the shuffle version
        if (!ShuffleHeader.DEFAULT_HTTP_HEADER_NAME
                .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_NAME))
                || !ShuffleHeader.DEFAULT_HTTP_HEADER_VERSION
                        .equals(connection.getHeaderField(ShuffleHeader.HTTP_HEADER_VERSION))) {
            throw new IOException("Incompatible shuffle response version");
        }
        // get the replyHash which is HMac of the encHash we sent to the server
        String replyHash = connection.getHeaderField(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH);
        if (replyHash == null) {
            throw new IOException("security validation of TT Map output failed");
        }
        LOG.debug("url=" + msgToEncode + ";encHash=" + encHash + ";replyHash=" + replyHash);
        // verify that replyHash is HMac of encHash
        SecureShuffleUtils.verifyReply(replyHash, encHash, jobTokenSecret);
        LOG.info("for url=" + msgToEncode + " sent hash and receievd reply");
    } catch (IOException ie) {
        ioErrs.increment(1);
        LOG.warn("Failed to connect to " + host + " with " + remaining.size() + " map outputs", ie);

        // If connect did not succeed, just mark all the maps as failed,
        // indirectly penalizing the host
        if (!connectSucceeded) {
            for (InputAttemptIdentifier left : remaining) {
                scheduler.copyFailed(left, host, connectSucceeded);
            }
        } else {
            // If we got a read error at this stage, it implies there was a problem
            // with the first map, typically lost map. So, penalize only that map
            // and add the rest
            InputAttemptIdentifier firstMap = srcAttempts.get(0);
            scheduler.copyFailed(firstMap, host, connectSucceeded);
        }

        // Add back all the remaining maps, WITHOUT marking them as failed
        for (InputAttemptIdentifier left : remaining) {
            // TODO Should the first one be skipped ?
            scheduler.putBackKnownMapOutput(host, left);
        }

        return;
    }

    try {
        // Loop through available map-outputs and fetch them
        // On any error, faildTasks is not null and we exit
        // after putting back the remaining maps to the 
        // yet_to_be_fetched list and marking the failed tasks.
        InputAttemptIdentifier[] failedTasks = null;
        while (!remaining.isEmpty() && failedTasks == null) {
            failedTasks = copyMapOutput(host, input);
        }

        if (failedTasks != null && failedTasks.length > 0) {
            LOG.warn("copyMapOutput failed for tasks " + Arrays.toString(failedTasks));
            for (InputAttemptIdentifier left : failedTasks) {
                scheduler.copyFailed(left, host, true);
            }
        }

        IOUtils.cleanup(LOG, input);

        // Sanity check
        if (failedTasks == null && !remaining.isEmpty()) {
            throw new IOException(
                    "server didn't return all expected map outputs: " + remaining.size() + " left.");
        }
    } finally {
        for (InputAttemptIdentifier left : remaining) {
            scheduler.putBackKnownMapOutput(host, left);
        }
    }
}

From source file:iracing.webapi.IracingWebApi.java

private boolean forumLoginAndGetCookie() {
    try {/*  w w w  .j  a  v  a  2s.  com*/
        // Make a connect to the server
        URL url = new URL(FORUM_LOGIN_URL);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();

        conn.addRequestProperty(COOKIE, cookie);

        conn.setDoInput(true);
        conn.setUseCaches(false);
        conn.setInstanceFollowRedirects(false);
        HttpURLConnection.setFollowRedirects(false);

        conn.connect();

        if (isMaintenancePage(conn))
            return false;

        String headerName;
        boolean containsCookie = false;
        for (int i = 1; (headerName = conn.getHeaderFieldKey(i)) != null; i++) {
            if (headerName.equalsIgnoreCase(SET_COOKIE)) {
                containsCookie = true;
                addToCookieMap(conn.getHeaderField(i));
            }
        }
        if (containsCookie)
            createCookieFromMap();

        conn.disconnect();
    } catch (Exception ex) {
        ex.printStackTrace();
        return false;
    }
    return true;
}

From source file:com.esri.gpt.control.arcims.ServletConnectorProxy.java

/**
 * Communicates with redirect url and works as a transparent proxy
 * /* w  w  w  .j ava  2s  .  c om*/
 * @param request
 *          the servlet request
 * @param response
 *          the servlet response
 * @throws IOException
 *           if an exception occurs
 */
private void executeProxy(HttpServletRequest request, HttpServletResponse response) throws IOException {

    HttpURLConnection httpCon = null;
    URL redirectURL = null;
    InputStream input = null;
    OutputStream output = null;
    InputStream proxyInput = null;
    OutputStream proxyOutput = null;

    try {

        input = request.getInputStream();
        output = response.getOutputStream();

        String sQueryStr = request.getQueryString();
        String sAuthorization = request.getHeader("Authorization");
        String requestBody = readInputCharacters(input);
        String requestMethod = request.getMethod();
        String contentType = request.getContentType();
        String encoding = request.getCharacterEncoding();

        LOGGER.finer(" Request method = " + requestMethod);
        LOGGER.finer(" Query string = " + sQueryStr);
        LOGGER.finer(" Authorization header =" + sAuthorization);
        LOGGER.finer(" Character Encoding = " + encoding);
        LOGGER.finer(" The redirect URL is " + this._redirectURL + "?" + sQueryStr);

        redirectURL = new URL(this._redirectURL + "?" + sQueryStr);

        httpCon = (HttpURLConnection) redirectURL.openConnection();

        httpCon.setDoInput(true);
        httpCon.setDoOutput(true);
        httpCon.setUseCaches(false);
        httpCon.setRequestMethod(requestMethod);
        httpCon.setRequestProperty("Content-type", contentType);

        if (sAuthorization != null) {
            httpCon.addRequestProperty("Authorization", sAuthorization);
        }

        proxyOutput = httpCon.getOutputStream();
        send(requestBody, proxyOutput);

        String authenticateHdr = httpCon.getHeaderField("WWW-Authenticate");
        if (authenticateHdr != null) {
            LOGGER.finer(" WWW-Authenticate : " + authenticateHdr);
            response.setHeader("WWW-Authenticate",
                    StringEscapeUtils.escapeHtml4(Val.stripControls(authenticateHdr)));
        }
        LOGGER.finer(" Response Code : " + httpCon.getResponseCode());

        if ((httpCon.getResponseCode() == HttpURLConnection.HTTP_FORBIDDEN)) {

            response.sendError(HttpServletResponse.SC_FORBIDDEN);
        } else if ((httpCon.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED)) {
            response.sendError(HttpServletResponse.SC_UNAUTHORIZED);
        } else if ((httpCon.getResponseCode() == HttpURLConnection.HTTP_INTERNAL_ERROR)) {
            response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        } else {
            proxyInput = httpCon.getInputStream();
            send(proxyInput, output);
        }

    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (input != null) {
            input.close();
        }
        if (output != null) {
            output.close();
        }
        if (proxyInput != null) {
            proxyInput.close();
        }
        if (proxyOutput != null) {
            proxyOutput.close();
        }
        if (httpCon != null) {
            httpCon.disconnect();
        }
    }
}

From source file:org.apache.zeppelin.warp10.Warp10Interpreter.java

public Pair<InterpreterResult.Code, String> execRequest(String body) throws Exception {

    ///*from   w w w .j a va  2s  . c om*/
    // Execute the request on current url defined
    //

    String url = this.current_Url;
    url += "/exec";
    URL obj = new URL(url);
    HttpURLConnection con = null;

    //
    // If HTTPS execute an HTTPS connection
    //

    if (url.startsWith("https")) {
        con = (HttpsURLConnection) obj.openConnection();
    } else {
        con = (HttpURLConnection) obj.openConnection();
    }

    //add request header
    con.setDoOutput(true);
    con.setDoInput(true);
    con.setRequestMethod("POST");
    con.setChunkedStreamingMode(16384);
    con.connect();

    //
    // Write the body in the request
    //

    OutputStream os = con.getOutputStream();
    //GZIPOutputStream out = new GZIPOutputStream(os);
    PrintWriter pw = new PrintWriter(os);
    pw.println(body);
    pw.close();

    StringBuffer response = new StringBuffer();
    Pair<InterpreterResult.Code, String> resultPair = null;

    //
    // If answer equals 200 parse result stream, otherwise error Stream
    //

    if (200 == con.getResponseCode()) {
        BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
        String inputLine;

        while ((inputLine = in.readLine()) != null) {
            response.append(inputLine);
        }
        resultPair = new Pair<InterpreterResult.Code, String>(InterpreterResult.Code.SUCCESS,
                response.toString());
        in.close();
        con.disconnect();
    } else {
        String fillBackEnd = "Warp10";
        if (this.isCzdBackend) {
            fillBackEnd = "CityzenData";
        }
        String errorLine = "\"Error-Line\":" + con.getHeaderField("X-" + fillBackEnd + "-Error-Line");
        String errorMsg = "\"Error-Message\":\"" + con.getHeaderField("X-" + fillBackEnd + "-Error-Message")
                + "\"";
        response.append("[{");
        response.append(errorLine + ",");
        response.append(errorMsg);
        boolean getBody = (null == con.getContentType());
        if (!getBody && !con.getContentType().startsWith("text/html")) {
            getBody = true;
        }
        if (getBody) {
            response.append(",\"Body\":\"");
            BufferedReader in = new BufferedReader(new InputStreamReader(con.getErrorStream()));
            String inputLine;

            while ((inputLine = in.readLine()) != null) {
                response.append(inputLine);
            }
            in.close();
            response.append("\"");
        }
        response.append("}]");
        resultPair = new Pair<InterpreterResult.Code, String>(InterpreterResult.Code.ERROR,
                response.toString());
        con.disconnect();
    }

    //
    // Return the body message with its associated code (SUCESS or ERROR)
    //

    return resultPair;
}

From source file:org.oclc.oai.harvester2.verb.HarvesterVerb.java

/**
 * Preforms the OAI request//from  w  w  w  .  j  a  va 2s  .c  om
 *
 * @param requestURL
 * @throws IOException
 * @throws ParserConfigurationException
 * @throws SAXException
 * @throws TransformerException
 */
public void harvestOldOclcImplementation(String requestURL)
        throws IOException, ParserConfigurationException, SAXException, TransformerException {
    this.requestURL = requestURL;
    logger.debug("requestURL=" + requestURL);
    InputStream in;
    URL url = new URL(requestURL);
    HttpURLConnection con;
    int responseCode;
    do {
        con = (HttpURLConnection) url.openConnection();
        con.setRequestProperty("User-Agent", "OAIHarvester/2.0");
        con.setRequestProperty("Accept-Encoding", "compress, gzip, identify");
        try {
            responseCode = con.getResponseCode();
            logger.debug("responseCode=" + responseCode);
        } catch (FileNotFoundException e) {
            // assume it's a 503 response
            logger.error(requestURL, e);
            responseCode = HttpURLConnection.HTTP_UNAVAILABLE;
        }

        if (responseCode == HttpURLConnection.HTTP_UNAVAILABLE) {
            long retrySeconds = con.getHeaderFieldInt("Retry-After", -1);
            if (retrySeconds == -1) {
                long now = (new Date()).getTime();
                long retryDate = con.getHeaderFieldDate("Retry-After", now);
                retrySeconds = retryDate - now;
            }
            if (retrySeconds == 0) { // Apparently, it's a bad URL
                throw new FileNotFoundException("Bad URL?");
            }
            logger.warn("Server response: Retry-After=" + retrySeconds);
            if (retrySeconds > 0) {
                try {
                    Thread.sleep(retrySeconds * 1000);
                } catch (InterruptedException ex) {
                    ex.printStackTrace();
                }
            }
        }
    } while (responseCode == HttpURLConnection.HTTP_UNAVAILABLE);
    String contentEncoding = con.getHeaderField("Content-Encoding");
    logger.debug("contentEncoding=" + contentEncoding);
    if ("compress".equals(contentEncoding)) {
        ZipInputStream zis = new ZipInputStream(con.getInputStream());
        zis.getNextEntry();
        in = zis;
    } else if ("gzip".equals(contentEncoding)) {
        in = new GZIPInputStream(con.getInputStream());
    } else if ("deflate".equals(contentEncoding)) {
        in = new InflaterInputStream(con.getInputStream());
    } else {
        in = con.getInputStream();
    }

    InputSource data = new InputSource(in);

    Thread t = Thread.currentThread();
    DocumentBuilder builder = (DocumentBuilder) builderMap.get(t);
    if (builder == null) {
        builder = factory.newDocumentBuilder();
        builderMap.put(t, builder);
    }
    doc = builder.parse(data);

    StringTokenizer tokenizer = new StringTokenizer(getSingleString("/*/@xsi:schemaLocation"), " ");
    StringBuffer sb = new StringBuffer();
    while (tokenizer.hasMoreTokens()) {
        if (sb.length() > 0)
            sb.append(" ");
        sb.append(tokenizer.nextToken());
    }
    this.schemaLocation = sb.toString();
}