Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.hive.hcatalog.templeton; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeoutException; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.Future; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class JobRequestExecutor<T> { private static final Logger LOG = LoggerFactory.getLogger(JobRequestExecutor.class); private static AppConfig appConf = Main.getAppConfigInstance(); /* * Thread pool to execute job requests. */ private ThreadPoolExecutor jobExecutePool = null; /* * Type of job request. */ private JobRequestType requestType; /* * Config name used to find the number of concurrent requests. */ private String concurrentRequestsConfigName; /* * Config name used to find the maximum time job request can be executed. */ private String jobTimeoutConfigName; /* * Job request execution time out in seconds. If it is 0 then request * will not be timed out. */ private int requestExecutionTimeoutInSec = 0; /* * Amount of time a thread can be alive in thread pool before cleaning this up. Core threads * will not be cleanup from thread pool. */ private int threadKeepAliveTimeInHours = 1; /* * Maximum number of times a cancel request is sent to job request execution * thread. Future.cancel may not be able to interrupt the thread if it is * blocked on network calls. */ private int maxTaskCancelRetryCount = 10; /* * Wait time in milliseconds before another cancel request is made. */ private int maxTaskCancelRetryWaitTimeInMs = 1000; /* * A flag to indicate whether to cancel the task when exception TimeoutException or * InterruptedException or CancellationException raised. The default is cancel thread. */ private boolean enableCancelTask = true; /* * Job Request type. */ public enum JobRequestType { Submit, Status, List } /* * Creates a job request object and sets up execution environment. Creates a thread pool * to execute job requests. * * @param requestType * Job request type * * @param concurrentRequestsConfigName * Config name to be used to extract number of concurrent requests to be serviced. * * @param jobTimeoutConfigName * Config name to be used to extract maximum time a task can execute a request. * * @param enableCancelTask * A flag to indicate whether to cancel the task when exception TimeoutException * or InterruptedException or CancellationException raised. * */ public JobRequestExecutor(JobRequestType requestType, String concurrentRequestsConfigName, String jobTimeoutConfigName, boolean enableCancelTask) { this.concurrentRequestsConfigName = concurrentRequestsConfigName; this.jobTimeoutConfigName = jobTimeoutConfigName; this.requestType = requestType; this.enableCancelTask = enableCancelTask; /* * The default number of threads will be 0. That means thread pool is not used and * operation is executed with the current thread. */ int threads = !StringUtils.isEmpty(concurrentRequestsConfigName) ? appConf.getInt(concurrentRequestsConfigName, 0) : 0; if (threads > 0) { /* * Create a thread pool with no queue wait time to execute the operation. This will ensure * that job requests are rejected if there are already maximum number of threads busy. */ this.jobExecutePool = new ThreadPoolExecutor(threads, threads, threadKeepAliveTimeInHours, TimeUnit.HOURS, new SynchronousQueue<Runnable>()); this.jobExecutePool.allowCoreThreadTimeOut(true); /* * Get the job request time out value. If this configuration value is set to 0 * then job request will wait until it finishes. */ if (!StringUtils.isEmpty(jobTimeoutConfigName)) { this.requestExecutionTimeoutInSec = appConf.getInt(jobTimeoutConfigName, 0); } LOG.info("Configured " + threads + " threads for job request type " + this.requestType + " with time out " + this.requestExecutionTimeoutInSec + " s."); } else { /* * If threads are not configured then they will be executed in current thread itself. */ LOG.info("No thread pool configured for job request type " + this.requestType); } } /* * Creates a job request object and sets up execution environment. Creates a thread pool * to execute job requests. * * @param requestType * Job request type * * @param concurrentRequestsConfigName * Config name to be used to extract number of concurrent requests to be serviced. * * @param jobTimeoutConfigName * Config name to be used to extract maximum time a task can execute a request. * */ public JobRequestExecutor(JobRequestType requestType, String concurrentRequestsConfigName, String jobTimeoutConfigName) { this(requestType, concurrentRequestsConfigName, jobTimeoutConfigName, true); } /* * Returns true of thread pool is created and can be used for executing a job request. * Otherwise, returns false. */ public boolean isThreadPoolEnabled() { return this.jobExecutePool != null; } /* * Executes job request operation. If thread pool is not created then job request is * executed in current thread itself. * * @param jobExecuteCallable * Callable object to run the job request task. * */ public T execute(JobCallable<T> jobExecuteCallable) throws InterruptedException, TimeoutException, TooManyRequestsException, ExecutionException { /* * The callable shouldn't be null to execute. The thread pool also should be configured * to execute requests. */ assert (jobExecuteCallable != null); assert (this.jobExecutePool != null); String type = this.requestType.toString().toLowerCase(); String retryMessageForConcurrentRequests = "Please wait for some time before retrying " + "the operation. Please refer to the config " + concurrentRequestsConfigName + " to configure concurrent requests."; LOG.debug("Starting new " + type + " job request with time out " + this.requestExecutionTimeoutInSec + "seconds."); Future<T> future = null; try { future = this.jobExecutePool.submit(jobExecuteCallable); } catch (RejectedExecutionException rejectedException) { /* * Not able to find thread to execute the job request. Raise Busy exception and client * can retry the operation. */ String tooManyRequestsExceptionMessage = "Unable to service the " + type + " job request as " + "templeton service is busy with too many " + type + " job requests. " + retryMessageForConcurrentRequests; LOG.warn(tooManyRequestsExceptionMessage); throw new TooManyRequestsException(tooManyRequestsExceptionMessage); } T result = null; try { result = this.requestExecutionTimeoutInSec > 0 ? future.get(this.requestExecutionTimeoutInSec, TimeUnit.SECONDS) : future.get(); } catch (TimeoutException e) { /* * See if the execution thread has just completed operation and result is available. * If result is available then return the result. Otherwise, raise exception. */ if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) { String message = this.requestType + " job request got timed out. Please wait for some time " + "before retrying the operation. Please refer to the config " + jobTimeoutConfigName + " to configure job request time out."; LOG.warn(message); /* * Throw TimeoutException to caller. */ throw new TimeoutException(message); } } catch (InterruptedException e) { /* * See if the execution thread has just completed operation and result is available. * If result is available then return the result. Otherwise, raise exception. */ if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) { String message = this.requestType + " job request got interrupted. Please wait for some time " + "before retrying the operation."; LOG.warn(message); /* * Throw TimeoutException to caller. */ throw new InterruptedException(message); } } catch (CancellationException e) { /* * See if the execution thread has just completed operation and result is available. * If result is available then return the result. Otherwise, raise exception. */ if ((result = tryGetJobResultOrSetJobStateFailed(jobExecuteCallable)) == null) { String message = this.requestType + " job request got cancelled and thread got interrupted. " + "Please wait for some time before retrying the operation."; LOG.warn(message); throw new InterruptedException(message); } } finally { /* * If the thread is still active and needs to be cancelled then cancel it. This may * happen in case task got interrupted, or timed out. */ if (enableCancelTask) { cancelExecutePoolThread(future); } } LOG.debug("Completed " + type + " job request."); return result; } /* * Initiate cancel request to cancel the thread execution and interrupt the thread. * If thread interruption is not handled by jobExecuteCallable then thread may continue * running to completion. The cancel call may fail for some scenarios. In that case, * retry the cancel call until it returns true or max retry count is reached. * * @param future * Future object which has handle to cancel the thread. * */ private void cancelExecutePoolThread(Future<T> future) { int retryCount = 0; while (retryCount < this.maxTaskCancelRetryCount && !future.isDone()) { LOG.info("Task is still executing the job request. Cancelling it with retry count: " + retryCount); if (future.cancel(true)) { /* * Cancelled the job request and return to client. */ LOG.info("Cancel job request issued successfully."); return; } retryCount++; try { Thread.sleep(this.maxTaskCancelRetryWaitTimeInMs); } catch (InterruptedException e) { /* * Nothing to do. Just retry. */ } } LOG.warn("Failed to cancel the job. isCancelled: " + future.isCancelled() + " Retry count: " + retryCount); } /* * Tries to get the job result if job request is completed. Otherwise it sets job status * to FAILED such that execute thread can do necessary clean up based on FAILED state. */ private T tryGetJobResultOrSetJobStateFailed(JobCallable<T> jobExecuteCallable) { if (!jobExecuteCallable.setJobStateFailed()) { LOG.info("Job is already COMPLETED. Returning the result."); return jobExecuteCallable.returnResult; } else { LOG.info("Job status set to FAILED. Job clean up to be done by execute thread " + "after job request is executed."); return null; } } }