Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.fs.azure; import java.net.HttpURLConnection; import java.util.Date; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import com.microsoft.azure.storage.OperationContext; import com.microsoft.azure.storage.RequestResult; import com.microsoft.azure.storage.ResponseReceivedEvent; import com.microsoft.azure.storage.SendingRequestEvent; import com.microsoft.azure.storage.StorageEvent; /* * Self throttling is implemented by hooking into send & response callbacks * One instance of this class is created per operationContext so each blobUpload/blobDownload/etc. * * Self throttling only applies to 2nd and subsequent packets of an operation. This is a simple way to * ensure it only affects bulk transfers and not every tiny request. * * A blobDownload will involve sequential packet transmissions and so there are no concurrency concerns * A blobUpload will generally involve concurrent upload worker threads that share one operationContext and one throttling instance. * -- we do not track the latencies for each worker thread as they are doing similar work and will rarely collide in practice. * -- concurrent access to lastE2Edelay must be protected. * -- volatile is necessary and should be sufficient to protect simple access to primitive values (java 1.5 onwards) * -- synchronized{} blocks are also used to be conservative and for easier maintenance. * * If an operation were to perform concurrent GETs and PUTs there is the possibility of getting confused regarding * whether lastE2Edelay was a read or write measurement. This scenario does not occur. * * readFactor = target read throughput as factor of unrestricted throughput. * writeFactor = target write throughput as factor of unrestricted throughput. * * As we introduce delays it is important to only measure the actual E2E latency and not the augmented latency * To achieve this, we fiddle the 'startDate' of the transfer tracking object. */ /** * * Introduces delays in our Azure traffic to prevent overrunning the server-side throttling limits. * */ @InterfaceAudience.Private public class SelfThrottlingIntercept { public static final Log LOG = LogFactory.getLog(SelfThrottlingIntercept.class); private final float readFactor; private final float writeFactor; private final OperationContext operationContext; // Concurrency: access to non-final members must be thread-safe private long lastE2Elatency; public SelfThrottlingIntercept(OperationContext operationContext, float readFactor, float writeFactor) { this.operationContext = operationContext; this.readFactor = readFactor; this.writeFactor = writeFactor; } public static void hook(OperationContext operationContext, float readFactor, float writeFactor) { SelfThrottlingIntercept throttler = new SelfThrottlingIntercept(operationContext, readFactor, writeFactor); ResponseReceivedListener responseListener = throttler.new ResponseReceivedListener(); SendingRequestListener sendingListener = throttler.new SendingRequestListener(); operationContext.getResponseReceivedEventHandler().addListener(responseListener); operationContext.getSendingRequestEventHandler().addListener(sendingListener); } public void responseReceived(ResponseReceivedEvent event) { RequestResult result = event.getRequestResult(); Date startDate = result.getStartDate(); Date stopDate = result.getStopDate(); long elapsed = stopDate.getTime() - startDate.getTime(); synchronized (this) { this.lastE2Elatency = elapsed; } if (LOG.isDebugEnabled()) { int statusCode = result.getStatusCode(); String etag = result.getEtag(); HttpURLConnection urlConnection = (HttpURLConnection) event.getConnectionObject(); int contentLength = urlConnection.getContentLength(); String requestMethod = urlConnection.getRequestMethod(); long threadId = Thread.currentThread().getId(); LOG.debug(String.format( "SelfThrottlingIntercept:: ResponseReceived: threadId=%d, Status=%d, Elapsed(ms)=%d, ETAG=%s, contentLength=%d, requestMethod=%s", threadId, statusCode, elapsed, etag, contentLength, requestMethod)); } } public void sendingRequest(SendingRequestEvent sendEvent) { long lastLatency; boolean operationIsRead; // for logging synchronized (this) { lastLatency = this.lastE2Elatency; } float sleepMultiple; HttpURLConnection urlConnection = (HttpURLConnection) sendEvent.getConnectionObject(); // Azure REST API never uses POST, so PUT is a sufficient test for an // upload. if (urlConnection.getRequestMethod().equalsIgnoreCase("PUT")) { operationIsRead = false; sleepMultiple = (1 / writeFactor) - 1; } else { operationIsRead = true; sleepMultiple = (1 / readFactor) - 1; } long sleepDuration = (long) (sleepMultiple * lastLatency); if (sleepDuration < 0) { sleepDuration = 0; } if (sleepDuration > 0) { try { // Thread.sleep() is not exact but it seems sufficiently accurate for // our needs. If needed this could become a loop of small waits that // tracks actual // elapsed time. Thread.sleep(sleepDuration); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } // reset to avoid counting the sleep against request latency sendEvent.getRequestResult().setStartDate(new Date()); } if (LOG.isDebugEnabled()) { boolean isFirstRequest = (lastLatency == 0); long threadId = Thread.currentThread().getId(); LOG.debug(String.format( " SelfThrottlingIntercept:: SendingRequest: threadId=%d, requestType=%s, isFirstRequest=%b, sleepDuration=%d", threadId, operationIsRead ? "read " : "write", isFirstRequest, sleepDuration)); } } // simply forwards back to the main class. // this is necessary as our main class cannot implement two base-classes. @InterfaceAudience.Private class SendingRequestListener extends StorageEvent<SendingRequestEvent> { @Override public void eventOccurred(SendingRequestEvent event) { sendingRequest(event); } } // simply forwards back to the main class. // this is necessary as our main class cannot implement two base-classes. @InterfaceAudience.Private class ResponseReceivedListener extends StorageEvent<ResponseReceivedEvent> { @Override public void eventOccurred(ResponseReceivedEvent event) { responseReceived(event); } } }