Example usage for java.util.concurrent ScheduledExecutorService awaitTermination

List of usage examples for java.util.concurrent ScheduledExecutorService awaitTermination

Introduction

In this page you can find the example usage for java.util.concurrent ScheduledExecutorService awaitTermination.

Prototype

boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException;

Source Link

Document

Blocks until all tasks have completed execution after a shutdown request, or the timeout occurs, or the current thread is interrupted, whichever happens first.

Usage

From source file:org.elasticsearch.client.sniff.SnifferTests.java

public void testTaskCancelling() throws Exception {
    RestClient restClient = mock(RestClient.class);
    HostsSniffer hostsSniffer = mock(HostsSniffer.class);
    Scheduler noOpScheduler = new Scheduler() {
        @Override/*from  w  w  w .  j a v  a2s . com*/
        public Future<?> schedule(Sniffer.Task task, long delayMillis) {
            return null;
        }

        @Override
        public void shutdown() {
        }
    };
    Sniffer sniffer = new Sniffer(restClient, hostsSniffer, noOpScheduler, 0L, 0L);
    ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    try {
        int numIters = randomIntBetween(50, 100);
        for (int i = 0; i < numIters; i++) {
            Sniffer.Task task = sniffer.new Task(0L);
            TaskWrapper wrapper = new TaskWrapper(task);
            Future<?> future;
            if (rarely()) {
                future = executor.schedule(wrapper, randomLongBetween(0L, 200L), TimeUnit.MILLISECONDS);
            } else {
                future = executor.submit(wrapper);
            }
            Sniffer.ScheduledTask scheduledTask = new Sniffer.ScheduledTask(task, future);
            boolean skip = scheduledTask.skip();
            try {
                assertNull(future.get());
            } catch (CancellationException ignore) {
                assertTrue(future.isCancelled());
            }

            if (skip) {
                //the task was either cancelled before starting, in which case it will never start (thanks to Future#cancel),
                //or skipped, in which case it will run but do nothing (thanks to Task#skip).
                //Here we want to make sure that whenever skip returns true, the task either won't run or it won't do anything,
                //otherwise we may end up with parallel sniffing tracks given that each task schedules the following one. We need to
                // make sure that onFailure takes scheduling over while at the same time ordinary rounds don't go on.
                assertFalse(task.hasStarted());
                assertTrue(task.isSkipped());
                assertTrue(future.isCancelled());
                assertTrue(future.isDone());
            } else {
                //if a future is cancelled when its execution has already started, future#get throws CancellationException before
                //completion. The execution continues though so we use a latch to try and wait for the task to be completed.
                //Here we want to make sure that whenever skip returns false, the task will be completed, otherwise we may be
                //missing to schedule the following round, which means no sniffing will ever happen again besides on failure sniffing.
                assertTrue(wrapper.await());
                //the future may or may not be cancelled but the task has for sure started and completed
                assertTrue(task.toString(), task.hasStarted());
                assertFalse(task.isSkipped());
                assertTrue(future.isDone());
            }
            //subsequent cancel calls return false for sure
            int cancelCalls = randomIntBetween(1, 10);
            for (int j = 0; j < cancelCalls; j++) {
                assertFalse(scheduledTask.skip());
            }
        }
    } finally {
        executor.shutdown();
        executor.awaitTermination(1000, TimeUnit.MILLISECONDS);
    }
}

From source file:org.apache.ambari.server.bootstrap.BSRunner.java

@Override
public void run() {

    if (sshHostInfo.getSshKey() == null || sshHostInfo.getSshKey().equals("")) {
        beforeBootStrap(sshHostInfo);/*from  ww w .  j  a  va 2 s. c om*/
    }

    String hostString = createHostString(sshHostInfo.getHosts());
    String user = sshHostInfo.getUser();
    String userRunAs = sshHostInfo.getUserRunAs();
    if (user == null || user.isEmpty()) {
        user = DEFAULT_USER;
    }
    String command[] = new String[12];
    BSStat stat = BSStat.RUNNING;
    String scriptlog = "";
    try {
        createRunDir();
        if (LOG.isDebugEnabled()) {
            // FIXME needs to be removed later
            // security hole
            LOG.debug("Using ssh key=\"" + sshHostInfo.getSshKey() + "\"");
        }

        String password = sshHostInfo.getPassword();
        if (password != null && !password.isEmpty()) {
            this.passwordFile = new File(this.requestIdDir, "host_pass");
            // TODO : line separator should be changed
            // if we are going to support multi platform server-agent solution
            String lineSeparator = System.getProperty("line.separator");
            password = password + lineSeparator;
            writePasswordFile(password);
        }

        writeSshKeyFile(sshHostInfo.getSshKey());
        /* Running command:
         * script hostlist bsdir user sshkeyfile
         */
        command[0] = this.bsScript;
        command[1] = hostString;
        command[2] = this.requestIdDir.toString();
        command[3] = user;
        command[4] = this.sshKeyFile.toString();
        command[5] = this.agentSetupScript.toString();
        command[6] = this.ambariHostname;
        command[7] = this.clusterOsFamily;
        command[8] = this.projectVersion;
        command[9] = this.serverPort + "";
        command[10] = userRunAs;
        command[11] = (this.passwordFile == null) ? "null" : this.passwordFile.toString();
        LOG.info("Host= " + hostString + " bs=" + this.bsScript + " requestDir=" + requestIdDir + " user="
                + user + " keyfile=" + this.sshKeyFile + " passwordFile " + this.passwordFile + " server="
                + this.ambariHostname + " version=" + projectVersion + " serverPort=" + this.serverPort
                + " userRunAs=" + userRunAs);

        String[] env = new String[] { "AMBARI_PASSPHRASE=" + agentSetupPassword };
        if (this.verbose)
            env = new String[] { env[0], " BS_VERBOSE=\"-vvv\" " };

        if (LOG.isDebugEnabled()) {
            LOG.debug(Arrays.toString(command));
        }

        String bootStrapOutputFilePath = requestIdDir + File.separator + "bootstrap.out";
        String bootStrapErrorFilePath = requestIdDir + File.separator + "bootstrap.err";

        Process process = Runtime.getRuntime().exec(command, env);

        PrintWriter stdOutWriter = null;
        PrintWriter stdErrWriter = null;

        try {
            stdOutWriter = new PrintWriter(bootStrapOutputFilePath);
            stdErrWriter = new PrintWriter(bootStrapErrorFilePath);
            IOUtils.copy(process.getInputStream(), stdOutWriter);
            IOUtils.copy(process.getErrorStream(), stdErrWriter);
        } finally {
            if (stdOutWriter != null)
                stdOutWriter.close();

            if (stdErrWriter != null)
                stdErrWriter.close();
        }

        // Startup a scheduled executor service to look through the logs
        ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
        BSStatusCollector statusCollector = new BSStatusCollector();
        ScheduledFuture<?> handle = scheduler.scheduleWithFixedDelay(statusCollector, 0, 10, TimeUnit.SECONDS);
        LOG.info("Kicking off the scheduler for polling on logs in " + this.requestIdDir);
        try {

            LOG.info("Bootstrap output, log=" + bootStrapErrorFilePath + " " + bootStrapOutputFilePath);
            int exitCode = process.waitFor();
            String outMesg = "";
            String errMesg = "";
            try {
                outMesg = FileUtils.readFileToString(new File(bootStrapOutputFilePath));
                errMesg = FileUtils.readFileToString(new File(bootStrapErrorFilePath));
            } catch (IOException io) {
                LOG.info("Error in reading files ", io);
            }
            scriptlog = outMesg + "\n\n" + errMesg;
            LOG.info("Script log Mesg " + scriptlog);
            if (exitCode != 0) {
                stat = BSStat.ERROR;
            } else {
                stat = BSStat.SUCCESS;
            }

            scheduler.schedule(new BSStatusCollector(), 0, TimeUnit.SECONDS);
            long startTime = System.currentTimeMillis();
            while (true) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Waiting for hosts status to be updated");
                }
                boolean pendingHosts = false;
                BootStrapStatus tmpStatus = bsImpl.getStatus(requestId);
                List<BSHostStatus> hostStatusList = tmpStatus.getHostsStatus();
                if (hostStatusList != null) {
                    for (BSHostStatus status : hostStatusList) {
                        if (status.getStatus().equals("RUNNING")) {
                            pendingHosts = true;
                        }
                    }
                } else {
                    //Failed to get host status, waiting for hosts status to be updated
                    pendingHosts = true;
                }
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Whether hosts status yet to be updated, pending=" + pendingHosts);
                }
                if (!pendingHosts) {
                    break;
                }
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e) {
                    // continue
                }
                long now = System.currentTimeMillis();
                if (now >= (startTime + 15000)) {
                    LOG.warn("Gave up waiting for hosts status to be updated");
                    break;
                }
            }
        } catch (InterruptedException e) {
            throw new IOException(e);
        } finally {
            handle.cancel(true);
            /* schedule a last update */
            scheduler.schedule(new BSStatusCollector(), 0, TimeUnit.SECONDS);
            scheduler.shutdownNow();
            try {
                scheduler.awaitTermination(10, TimeUnit.SECONDS);
            } catch (InterruptedException e) {
                LOG.info("Interruped while waiting for scheduler");
            }
            process.destroy();
        }
    } catch (IOException io) {
        LOG.info("Error executing bootstrap " + io.getMessage());
        stat = BSStat.ERROR;
    } finally {
        /* get the bstatus */
        BootStrapStatus tmpStatus = bsImpl.getStatus(requestId);
        List<BSHostStatus> hostStatusList = tmpStatus.getHostsStatus();
        if (hostStatusList != null) {
            for (BSHostStatus hostStatus : hostStatusList) {
                if ("FAILED".equals(hostStatus.getStatus())) {
                    stat = BSStat.ERROR;
                    break;
                }
            }
        } else {
            stat = BSStat.ERROR;
        }
        tmpStatus.setLog(scriptlog);
        tmpStatus.setStatus(stat);
        bsImpl.updateStatus(requestId, tmpStatus);
        bsImpl.reset();
        // Remove private ssh key after bootstrap is complete
        try {
            FileUtils.forceDelete(sshKeyFile);
        } catch (IOException io) {
            LOG.warn(io.getMessage());
        }
        if (passwordFile != null) {
            // Remove password file after bootstrap is complete
            try {
                FileUtils.forceDelete(passwordFile);
            } catch (IOException io) {
                LOG.warn(io.getMessage());
            }
        }
        finished();
    }
}

From source file:org.elasticsearch.client.sniff.SnifferTests.java

/**
 * Test behaviour when a bunch of onFailure sniffing rounds are triggered in parallel. Each run will always
 * schedule a subsequent afterFailure round. Also, for each onFailure round that starts, the net scheduled round
 * (either afterFailure or ordinary) gets cancelled.
 *///from  w  w  w  . j a  v a 2s . com
public void testSniffOnFailure() throws Exception {
    RestClient restClient = mock(RestClient.class);
    CountingHostsSniffer hostsSniffer = new CountingHostsSniffer();
    final AtomicBoolean initializing = new AtomicBoolean(true);
    final long sniffInterval = randomLongBetween(1, Long.MAX_VALUE);
    final long sniffAfterFailureDelay = randomLongBetween(1, Long.MAX_VALUE);
    int minNumOnFailureRounds = randomIntBetween(5, 10);
    final CountDownLatch initializingLatch = new CountDownLatch(1);
    final Set<Sniffer.ScheduledTask> ordinaryRoundsTasks = new CopyOnWriteArraySet<>();
    final AtomicReference<Future<?>> initializingFuture = new AtomicReference<>();
    final Set<Sniffer.ScheduledTask> onFailureTasks = new CopyOnWriteArraySet<>();
    final Set<Sniffer.ScheduledTask> afterFailureTasks = new CopyOnWriteArraySet<>();
    final AtomicBoolean onFailureCompleted = new AtomicBoolean(false);
    final CountDownLatch completionLatch = new CountDownLatch(1);
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    try {
        Scheduler scheduler = new Scheduler() {
            @Override
            public Future<?> schedule(final Sniffer.Task task, long delayMillis) {
                if (initializing.compareAndSet(true, false)) {
                    assertEquals(0L, delayMillis);
                    Future<?> future = executor.submit(new Runnable() {
                        @Override
                        public void run() {
                            try {
                                task.run();
                            } finally {
                                //we need to make sure that the sniffer is initialized, so the sniffOnFailure
                                //call does what it needs to do. Otherwise nothing happens until initialized.
                                initializingLatch.countDown();
                            }
                        }
                    });
                    assertTrue(initializingFuture.compareAndSet(null, future));
                    return future;
                }
                if (delayMillis == 0L) {
                    Future<?> future = executor.submit(task);
                    onFailureTasks.add(new Sniffer.ScheduledTask(task, future));
                    return future;
                }
                if (delayMillis == sniffAfterFailureDelay) {
                    Future<?> future = scheduleOrSubmit(task);
                    afterFailureTasks.add(new Sniffer.ScheduledTask(task, future));
                    return future;
                }

                assertEquals(sniffInterval, delayMillis);
                assertEquals(sniffInterval, task.nextTaskDelay);

                if (onFailureCompleted.get() && onFailureTasks.size() == afterFailureTasks.size()) {
                    completionLatch.countDown();
                    return mock(Future.class);
                }

                Future<?> future = scheduleOrSubmit(task);
                ordinaryRoundsTasks.add(new Sniffer.ScheduledTask(task, future));
                return future;
            }

            private Future<?> scheduleOrSubmit(Sniffer.Task task) {
                if (randomBoolean()) {
                    return executor.schedule(task, randomLongBetween(0L, 200L), TimeUnit.MILLISECONDS);
                } else {
                    return executor.submit(task);
                }
            }

            @Override
            public void shutdown() {
            }
        };
        final Sniffer sniffer = new Sniffer(restClient, hostsSniffer, scheduler, sniffInterval,
                sniffAfterFailureDelay);
        assertTrue("timeout waiting for sniffer to get initialized",
                initializingLatch.await(1000, TimeUnit.MILLISECONDS));

        ExecutorService onFailureExecutor = Executors.newFixedThreadPool(randomIntBetween(5, 20));
        Set<Future<?>> onFailureFutures = new CopyOnWriteArraySet<>();
        try {
            //with tasks executing quickly one after each other, it is very likely that the onFailure round gets skipped
            //as another round is already running. We retry till enough runs get through as that's what we want to test.
            while (onFailureTasks.size() < minNumOnFailureRounds) {
                onFailureFutures.add(onFailureExecutor.submit(new Runnable() {
                    @Override
                    public void run() {
                        sniffer.sniffOnFailure();
                    }
                }));
            }
            assertThat(onFailureFutures.size(), greaterThanOrEqualTo(minNumOnFailureRounds));
            for (Future<?> onFailureFuture : onFailureFutures) {
                assertNull(onFailureFuture.get());
            }
            onFailureCompleted.set(true);
        } finally {
            onFailureExecutor.shutdown();
            onFailureExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);
        }

        assertFalse(initializingFuture.get().isCancelled());
        assertTrue(initializingFuture.get().isDone());
        assertNull(initializingFuture.get().get());

        assertTrue("timeout waiting for sniffing rounds to be completed",
                completionLatch.await(1000, TimeUnit.MILLISECONDS));
        assertThat(onFailureTasks.size(), greaterThanOrEqualTo(minNumOnFailureRounds));
        assertEquals(onFailureTasks.size(), afterFailureTasks.size());

        for (Sniffer.ScheduledTask onFailureTask : onFailureTasks) {
            assertFalse(onFailureTask.future.isCancelled());
            assertTrue(onFailureTask.future.isDone());
            assertNull(onFailureTask.future.get());
            assertTrue(onFailureTask.task.hasStarted());
            assertFalse(onFailureTask.task.isSkipped());
        }

        int cancelledTasks = 0;
        int completedTasks = onFailureTasks.size() + 1;
        for (Sniffer.ScheduledTask afterFailureTask : afterFailureTasks) {
            if (assertTaskCancelledOrCompleted(afterFailureTask)) {
                completedTasks++;
            } else {
                cancelledTasks++;
            }
        }

        assertThat(ordinaryRoundsTasks.size(), greaterThan(0));
        for (Sniffer.ScheduledTask task : ordinaryRoundsTasks) {
            if (assertTaskCancelledOrCompleted(task)) {
                completedTasks++;
            } else {
                cancelledTasks++;
            }
        }
        assertEquals(onFailureTasks.size(), cancelledTasks);

        assertEquals(completedTasks, hostsSniffer.runs.get());
        int setHostsRuns = hostsSniffer.runs.get() - hostsSniffer.failures.get() - hostsSniffer.emptyList.get();
        verify(restClient, times(setHostsRuns)).setHosts(Matchers.<HttpHost>anyVararg());
        verifyNoMoreInteractions(restClient);
    } finally {
        executor.shutdown();
        executor.awaitTermination(1000L, TimeUnit.MILLISECONDS);
    }
}

From source file:org.alfresco.repo.security.sync.TenantChainingUserRegistrySynchronizer.java

private void synchronizeInternal(boolean forceUpdate, boolean isFullSync, final boolean splitTxns) {
    TenantChainingUserRegistrySynchronizer.logger
            .debug("Running a sync for domain: " + SEIPTenantIntegration.getTenantId());
    if (TenantChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {

        if (forceUpdate) {
            TenantChainingUserRegistrySynchronizer.logger.debug("Running a full sync.");
        } else {/*  w  ww .  j a  v  a 2  s.  c o  m*/
            TenantChainingUserRegistrySynchronizer.logger.debug("Running a differential sync.");
        }
        if (allowDeletions) {
            TenantChainingUserRegistrySynchronizer.logger.debug("deletions are allowed");
        } else {
            TenantChainingUserRegistrySynchronizer.logger.debug("deletions are not allowed");
        }
        // Don't proceed with the sync if the repository is read only
        if (this.transactionService.isReadOnly()) {
            TenantChainingUserRegistrySynchronizer.logger
                    .warn("Unable to proceed with user registry synchronization. Repository is read only.");
            return;
        }
    }

    // Don't proceed with the sync if the repository is read only
    if (this.transactionService.isReadOnly()) {
        TenantChainingUserRegistrySynchronizer.logger
                .warn("Unable to proceed with user registry synchronization. Repository is read only.");
        return;
    }

    // Create a background executor that will refresh our lock. This means
    // we can request a lock with a relatively
    // small persistence time and not worry about it lasting after server
    // restarts. Note we use an independent
    // executor because this is a compound operation that spans accross
    // multiple batch processors.
    String lockToken = null;
    TraceableThreadFactory threadFactory = new TraceableThreadFactory();
    threadFactory.setNamePrefix("TenantChainingUserRegistrySynchronizer lock refresh");
    threadFactory.setThreadDaemon(true);
    ScheduledExecutorService lockRefresher = new ScheduledThreadPoolExecutor(1, threadFactory);

    // Let's ensure all exceptions get logged
    try {
        // First, try to obtain a lock to ensure we are the only node trying
        // to run this job
        try {
            if (splitTxns) {
                // If this is an automated sync on startup or scheduled
                // sync, don't even wait around for the lock.
                // Assume the sync will be completed on another node.
                lockToken = this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<String>() {
                            public String execute() throws Throwable {
                                return TenantChainingUserRegistrySynchronizer.this.jobLockService.getLock(
                                        TenantChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        TenantChainingUserRegistrySynchronizer.LOCK_TTL, 0, 1);
                            }
                        }, false, splitTxns);
            } else {
                // If this is a login-triggered sync, give it a few retries
                // before giving up
                lockToken = this.jobLockService.getLock(TenantChainingUserRegistrySynchronizer.LOCK_QNAME,
                        TenantChainingUserRegistrySynchronizer.LOCK_TTL, 3000, 10);
            }
        } catch (LockAcquisitionException e) {
            // Don't proceed with the sync if it is running on another node
            TenantChainingUserRegistrySynchronizer.logger.warn(
                    "User registry synchronization already running in another thread. Synchronize aborted");
            return;
        }

        // Schedule the lock refresh to run at regular intervals
        final String token = lockToken;
        lockRefresher.scheduleAtFixedRate(new Runnable() {
            public void run() {
                TenantChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<Object>() {
                            public Object execute() throws Throwable {
                                TenantChainingUserRegistrySynchronizer.this.jobLockService.refreshLock(token,
                                        TenantChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        TenantChainingUserRegistrySynchronizer.LOCK_TTL);
                                return null;
                            }
                        }, false, splitTxns);
            }
        }, TenantChainingUserRegistrySynchronizer.LOCK_TTL / 2,
                TenantChainingUserRegistrySynchronizer.LOCK_TTL / 2, TimeUnit.MILLISECONDS);

        Set<String> visitedZoneIds = new TreeSet<String>();
        Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

        // Work out the set of all zone IDs in the authentication chain so
        // that we can decide which users / groups
        // need 're-zoning'
        Set<String> allZoneIds = new TreeSet<String>();
        for (String id : instanceIds) {
            allZoneIds.add(AuthorityService.ZONE_AUTH_EXT_PREFIX + id);
        }
        for (String id : instanceIds) {
            ApplicationContext context = this.applicationContextManager.getApplicationContext(id);
            try {
                UserRegistry plugin = (UserRegistry) context.getBean(this.sourceBeanName);
                if (!(plugin instanceof ActivateableBean) || ((ActivateableBean) plugin).isActive()) {
                    if (TenantChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {
                        mbeanServer = (MBeanServerConnection) getApplicationContext()
                                .getBean("alfrescoMBeanServer");
                        try {
                            StringBuilder nameBuff = new StringBuilder(200).append(
                                    "Alfresco:Type=Configuration,Category=Authentication,id1=managed,id2=")
                                    .append(URLDecoder.decode(id, "UTF-8"));
                            ObjectName name = new ObjectName(nameBuff.toString());
                            if (mbeanServer != null && mbeanServer.isRegistered(name)) {
                                MBeanInfo info = mbeanServer.getMBeanInfo(name);
                                MBeanAttributeInfo[] attributes = info.getAttributes();
                                TenantChainingUserRegistrySynchronizer.logger.debug(id + " attributes:");
                                for (MBeanAttributeInfo attribute : attributes) {
                                    Object value = mbeanServer.getAttribute(name, attribute.getName());
                                    TenantChainingUserRegistrySynchronizer.logger
                                            .debug(attribute.getName() + " = " + value);
                                }
                            }
                        } catch (UnsupportedEncodingException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (MalformedObjectNameException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (InstanceNotFoundException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (IntrospectionException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (AttributeNotFoundException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (ReflectionException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (MBeanException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        } catch (IOException e) {
                            if (TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                                TenantChainingUserRegistrySynchronizer.logger.warn("Exception during logging",
                                        e);
                            }
                        }

                    }
                    if (TenantChainingUserRegistrySynchronizer.logger.isInfoEnabled()) {
                        TenantChainingUserRegistrySynchronizer.logger
                                .info("Synchronizing users and groups with user registry '" + id + "'");
                    }
                    if (isFullSync && TenantChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                        TenantChainingUserRegistrySynchronizer.logger
                                .warn("Full synchronization with user registry '" + id + "'");
                        if (allowDeletions) {
                            TenantChainingUserRegistrySynchronizer.logger.warn(
                                    "Some users and groups previously created by synchronization with this user registry may be removed.");
                        } else {
                            TenantChainingUserRegistrySynchronizer.logger.warn(
                                    "Deletions are disabled. Users and groups removed from this registry will be logged only and will remain in the repository. Users previously found in a different registry will be moved in the repository rather than recreated.");
                        }
                    }
                    // Work out whether we should do the work in a separate
                    // transaction (it's most performant if we
                    // bunch it into small transactions, but if we are doing
                    // a sync on login, it has to be the same
                    // transaction)
                    boolean requiresNew = splitTxns || AlfrescoTransactionSupport
                            .getTransactionReadState() == TxnReadState.TXN_READ_ONLY;

                    syncWithPlugin(id, plugin, forceUpdate, isFullSync, requiresNew, visitedZoneIds,
                            allZoneIds);
                }
            } catch (NoSuchBeanDefinitionException e) {
                // Ignore and continue
            }

        }
    } catch (RuntimeException e) {
        TenantChainingUserRegistrySynchronizer.logger.error("Synchronization aborted due to error", e);
        throw e;
    }
    // Release the lock if necessary
    finally {
        if (lockToken != null) {
            // Cancel the lock refresher
            // Because we may hit a perfect storm when trying to interrupt
            // workers in their unsynchronized getTask()
            // method we can't wait indefinitely and may have to retry the
            // shutdown
            int trys = 0;
            do {
                lockRefresher.shutdown();
                try {
                    lockRefresher.awaitTermination(TenantChainingUserRegistrySynchronizer.LOCK_TTL,
                            TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                }
            } while (!lockRefresher.isTerminated() && trys++ < 3);
            if (!lockRefresher.isTerminated()) {
                lockRefresher.shutdownNow();
                TenantChainingUserRegistrySynchronizer.logger.error("Failed to shut down lock refresher");
            }

            final String token = lockToken;
            this.transactionService.getRetryingTransactionHelper()
                    .doInTransaction(new RetryingTransactionCallback<Object>() {
                        public Object execute() throws Throwable {
                            TenantChainingUserRegistrySynchronizer.this.jobLockService.releaseLock(token,
                                    TenantChainingUserRegistrySynchronizer.LOCK_QNAME);
                            return null;
                        }
                    }, false, splitTxns);
        }
    }
}

From source file:org.alfresco.repo.security.sync.ChainingUserRegistrySynchronizer.java

private void synchronizeInternal(boolean forceUpdate, boolean isFullSync, final boolean splitTxns) {
    if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {

        if (forceUpdate) {
            ChainingUserRegistrySynchronizer.logger.debug("Running a full sync.");
        } else {//from w ww  .  j a v a 2 s .c o  m
            ChainingUserRegistrySynchronizer.logger.debug("Running a differential sync.");
        }
        if (allowDeletions) {
            ChainingUserRegistrySynchronizer.logger.debug("deletions are allowed");
        } else {
            ChainingUserRegistrySynchronizer.logger.debug("deletions are not allowed");
        }
        // Don't proceed with the sync if the repository is read only
        if (this.transactionService.isReadOnly()) {
            ChainingUserRegistrySynchronizer.logger
                    .warn("Unable to proceed with user registry synchronization. Repository is read only.");
            return;
        }
    }

    // Don't proceed with the sync if the repository is read only
    if (this.transactionService.isReadOnly()) {
        ChainingUserRegistrySynchronizer.logger
                .warn("Unable to proceed with user registry synchronization. Repository is read only.");
        return;
    }

    // Create a background executor that will refresh our lock. This means we can request a lock with a relatively
    // small persistence time and not worry about it lasting after server restarts. Note we use an independent
    // executor because this is a compound operation that spans accross multiple batch processors.
    String lockToken = null;
    TraceableThreadFactory threadFactory = new TraceableThreadFactory();
    threadFactory.setNamePrefix("ChainingUserRegistrySynchronizer lock refresh");
    threadFactory.setThreadDaemon(true);
    ScheduledExecutorService lockRefresher = new ScheduledThreadPoolExecutor(1, threadFactory);

    // Let's ensure all exceptions get logged
    try {
        // First, try to obtain a lock to ensure we are the only node trying to run this job
        try {
            if (splitTxns) {
                // If this is an automated sync on startup or scheduled sync, don't even wait around for the lock.
                // Assume the sync will be completed on another node.
                lockToken = this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<String>() {
                            public String execute() throws Throwable {
                                return ChainingUserRegistrySynchronizer.this.jobLockService.getLock(
                                        ChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        ChainingUserRegistrySynchronizer.LOCK_TTL, 0, 1);
                            }
                        }, false, splitTxns);
            } else {
                // If this is a login-triggered sync, give it a few retries before giving up
                lockToken = this.jobLockService.getLock(ChainingUserRegistrySynchronizer.LOCK_QNAME,
                        ChainingUserRegistrySynchronizer.LOCK_TTL, 3000, 10);
            }
        } catch (LockAcquisitionException e) {
            // Don't proceed with the sync if it is running on another node
            ChainingUserRegistrySynchronizer.logger.warn(
                    "User registry synchronization already running in another thread. Synchronize aborted");
            return;
        }

        // Schedule the lock refresh to run at regular intervals
        final String token = lockToken;
        lockRefresher.scheduleAtFixedRate(new Runnable() {
            public void run() {
                ChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<Object>() {
                            public Object execute() throws Throwable {
                                ChainingUserRegistrySynchronizer.this.jobLockService.refreshLock(token,
                                        ChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        ChainingUserRegistrySynchronizer.LOCK_TTL);
                                return null;
                            }
                        }, false, splitTxns);
            }
        }, ChainingUserRegistrySynchronizer.LOCK_TTL / 2, ChainingUserRegistrySynchronizer.LOCK_TTL / 2,
                TimeUnit.MILLISECONDS);

        Set<String> visitedZoneIds = new TreeSet<String>();
        Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

        // Work out the set of all zone IDs in the authentication chain so that we can decide which users / groups
        // need 're-zoning'
        Set<String> allZoneIds = new TreeSet<String>();
        for (String id : instanceIds) {
            allZoneIds.add(AuthorityService.ZONE_AUTH_EXT_PREFIX + id);
        }

        // Collect the plugins that we can sync : zoneId, plugin
        Map<String, UserRegistry> plugins = new HashMap<String, UserRegistry>();

        for (String id : instanceIds) {
            UserRegistry plugin;
            try {
                ApplicationContext context = this.applicationContextManager.getApplicationContext(id);
                plugin = (UserRegistry) context.getBean(this.sourceBeanName);
            } catch (RuntimeException e) {
                // The bean doesn't exist or this subsystem won't start. The reason would have been logged. Ignore and continue.
                continue;
            }

            if (!(plugin instanceof ActivateableBean) || ((ActivateableBean) plugin).isActive()) {
                // yes this plugin needs to be synced
                plugins.put(id, plugin);
            }
        }

        /**
         *  Sync starts here
         */
        notifySyncStart(plugins.keySet());

        for (String id : instanceIds) {
            UserRegistry plugin = plugins.get(id);

            if (plugin != null) {
                // If debug is enabled then dump out the contents of the authentication JMX bean
                if (ChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {
                    mbeanServer = (MBeanServerConnection) getApplicationContext()
                            .getBean("alfrescoMBeanServer");
                    try {
                        StringBuilder nameBuff = new StringBuilder(200)
                                .append("Alfresco:Type=Configuration,Category=Authentication,id1=managed,id2=")
                                .append(URLDecoder.decode(id, "UTF-8"));
                        ObjectName name = new ObjectName(nameBuff.toString());
                        if (mbeanServer != null && mbeanServer.isRegistered(name)) {
                            MBeanInfo info = mbeanServer.getMBeanInfo(name);
                            MBeanAttributeInfo[] attributes = info.getAttributes();
                            ChainingUserRegistrySynchronizer.logger.debug(id + " attributes:");
                            for (MBeanAttributeInfo attribute : attributes) {
                                Object value = mbeanServer.getAttribute(name, attribute.getName());
                                ChainingUserRegistrySynchronizer.logger
                                        .debug(attribute.getName() + " = " + value);
                            }
                        }
                    } catch (UnsupportedEncodingException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (MalformedObjectNameException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (InstanceNotFoundException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (IntrospectionException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (AttributeNotFoundException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (ReflectionException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (MBeanException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (IOException e) {
                        if (ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            ChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    }
                } // end of debug dump of active JMX bean
                if (ChainingUserRegistrySynchronizer.logger.isInfoEnabled()) {
                    ChainingUserRegistrySynchronizer.logger
                            .info("Synchronizing users and groups with user registry '" + id + "'");
                }
                if (isFullSync && ChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                    ChainingUserRegistrySynchronizer.logger
                            .warn("Full synchronization with user registry '" + id + "'");
                    if (allowDeletions) {
                        ChainingUserRegistrySynchronizer.logger.warn(
                                "Some users and groups previously created by synchronization with this user registry may be removed.");
                    } else {
                        ChainingUserRegistrySynchronizer.logger.warn(
                                "Deletions are disabled. Users and groups removed from this registry will be logged only and will remain in the repository. Users previously found in a different registry will be moved in the repository rather than recreated.");
                    }
                }
                // Work out whether we should do the work in a separate transaction (it's most performant if we
                // bunch it into small transactions, but if we are doing a sync on login, it has to be the same
                // transaction)
                boolean requiresNew = splitTxns
                        || AlfrescoTransactionSupport.getTransactionReadState() == TxnReadState.TXN_READ_ONLY;

                try {
                    /**
                     * Do the sync with the specified plugin
                     */
                    syncWithPlugin(id, plugin, forceUpdate, isFullSync, requiresNew, visitedZoneIds,
                            allZoneIds);

                    this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryEndEvent(this, id));
                } catch (final RuntimeException e) {
                    notifySyncDirectoryEnd(id, e);
                    throw e;
                }
            } // if plugin exists
        } // for each instanceId

        //End of successful synchronization here
        notifySyncEnd();
    } catch (final RuntimeException e) {
        notifySyncEnd(e);
        ChainingUserRegistrySynchronizer.logger.error("Synchronization aborted due to error", e);
        throw e;
    } finally {
        // Release the lock if necessary
        if (lockToken != null) {
            // Cancel the lock refresher
            // Because we may hit a perfect storm when trying to interrupt workers in their unsynchronized getTask()
            // method we can't wait indefinitely and may have to retry the shutdown
            int trys = 0;
            do {
                lockRefresher.shutdown();
                try {
                    lockRefresher.awaitTermination(ChainingUserRegistrySynchronizer.LOCK_TTL,
                            TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                }
            } while (!lockRefresher.isTerminated() && trys++ < 3);
            if (!lockRefresher.isTerminated()) {
                lockRefresher.shutdownNow();
                ChainingUserRegistrySynchronizer.logger.error("Failed to shut down lock refresher");
            }

            final String token = lockToken;
            this.transactionService.getRetryingTransactionHelper()
                    .doInTransaction(new RetryingTransactionCallback<Object>() {
                        public Object execute() throws Throwable {
                            ChainingUserRegistrySynchronizer.this.jobLockService.releaseLock(token,
                                    ChainingUserRegistrySynchronizer.LOCK_QNAME);
                            return null;
                        }
                    }, false, splitTxns);
        }
    }
}

From source file:org.cggh.repo.security.sync.CustomChainingUserRegistrySynchronizer.java

private void synchronizeInternal(boolean forceUpdate, boolean isFullSync, final boolean splitTxns) {
    if (CustomChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {

        if (forceUpdate) {
            CustomChainingUserRegistrySynchronizer.logger.debug("Running a full sync.");
        } else {//ww w . j  a  v a  2s . c  om
            CustomChainingUserRegistrySynchronizer.logger.debug("Running a differential sync.");
        }
        if (allowDeletions) {
            CustomChainingUserRegistrySynchronizer.logger.debug("deletions are allowed");
        } else {
            CustomChainingUserRegistrySynchronizer.logger.debug("deletions are not allowed");
        }
        // Don't proceed with the sync if the repository is read only
        if (this.transactionService.isReadOnly()) {
            CustomChainingUserRegistrySynchronizer.logger
                    .warn("Unable to proceed with user registry synchronization. Repository is read only.");
            return;
        }
    }

    // Don't proceed with the sync if the repository is read only
    if (this.transactionService.isReadOnly()) {
        CustomChainingUserRegistrySynchronizer.logger
                .warn("Unable to proceed with user registry synchronization. Repository is read only.");
        return;
    }

    // Create a background executor that will refresh our lock. This means we can request a lock with a relatively
    // small persistence time and not worry about it lasting after server restarts. Note we use an independent
    // executor because this is a compound operation that spans accross multiple batch processors.
    String lockToken = null;
    TraceableThreadFactory threadFactory = new TraceableThreadFactory();
    threadFactory.setNamePrefix("ChainingUserRegistrySynchronizer lock refresh");
    threadFactory.setThreadDaemon(true);
    ScheduledExecutorService lockRefresher = new ScheduledThreadPoolExecutor(1, threadFactory);

    // Let's ensure all exceptions get logged
    try {
        // First, try to obtain a lock to ensure we are the only node trying to run this job
        try {
            if (splitTxns) {
                // If this is an automated sync on startup or scheduled sync, don't even wait around for the lock.
                // Assume the sync will be completed on another node.
                lockToken = this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<String>() {
                            public String execute() throws Throwable {
                                return CustomChainingUserRegistrySynchronizer.this.jobLockService.getLock(
                                        CustomChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        CustomChainingUserRegistrySynchronizer.LOCK_TTL, 0, 1);
                            }
                        }, false, splitTxns);
            } else {
                // If this is a login-triggered sync, give it a few retries before giving up
                lockToken = this.jobLockService.getLock(CustomChainingUserRegistrySynchronizer.LOCK_QNAME,
                        CustomChainingUserRegistrySynchronizer.LOCK_TTL, 3000, 10);
            }
        } catch (LockAcquisitionException e) {
            // Don't proceed with the sync if it is running on another node
            CustomChainingUserRegistrySynchronizer.logger.warn(
                    "User registry synchronization already running in another thread. Synchronize aborted");
            return;
        }

        // Schedule the lock refresh to run at regular intervals
        final String token = lockToken;
        lockRefresher.scheduleAtFixedRate(new Runnable() {
            public void run() {
                CustomChainingUserRegistrySynchronizer.this.transactionService.getRetryingTransactionHelper()
                        .doInTransaction(new RetryingTransactionCallback<Object>() {
                            public Object execute() throws Throwable {
                                CustomChainingUserRegistrySynchronizer.this.jobLockService.refreshLock(token,
                                        CustomChainingUserRegistrySynchronizer.LOCK_QNAME,
                                        CustomChainingUserRegistrySynchronizer.LOCK_TTL);
                                return null;
                            }
                        }, false, splitTxns);
            }
        }, CustomChainingUserRegistrySynchronizer.LOCK_TTL / 2,
                CustomChainingUserRegistrySynchronizer.LOCK_TTL / 2, TimeUnit.MILLISECONDS);

        Set<String> visitedZoneIds = new TreeSet<String>();
        Collection<String> instanceIds = this.applicationContextManager.getInstanceIds();

        // Work out the set of all zone IDs in the authentication chain so that we can decide which users / groups
        // need 're-zoning'
        Set<String> allZoneIds = new TreeSet<String>();
        for (String id : instanceIds) {
            allZoneIds.add(AuthorityService.ZONE_AUTH_EXT_PREFIX + id);
        }

        // Collect the plugins that we can sync : zoneId, plugin
        Map<String, UserRegistry> plugins = new HashMap<String, UserRegistry>();

        for (String id : instanceIds) {
            UserRegistry plugin;
            try {
                ApplicationContext context = this.applicationContextManager.getApplicationContext(id);
                plugin = (UserRegistry) context.getBean(this.sourceBeanName);
            } catch (RuntimeException e) {
                // The bean doesn't exist or this subsystem won't start. The reason would have been logged. Ignore and continue.
                continue;
            }

            if (!(plugin instanceof ActivateableBean) || ((ActivateableBean) plugin).isActive()) {
                // yes this plugin needs to be synced
                plugins.put(id, plugin);
            }
        }

        /**
         *  Sync starts here
         */
        notifySyncStart(plugins.keySet());

        for (String id : instanceIds) {
            UserRegistry plugin = plugins.get(id);

            if (plugin != null) {
                // If debug is enabled then dump out the contents of the authentication JMX bean
                if (CustomChainingUserRegistrySynchronizer.logger.isDebugEnabled()) {
                    mbeanServer = (MBeanServerConnection) getApplicationContext()
                            .getBean("alfrescoMBeanServer");
                    try {
                        StringBuilder nameBuff = new StringBuilder(200)
                                .append("Alfresco:Type=Configuration,Category=Authentication,id1=managed,id2=")
                                .append(URLDecoder.decode(id, "UTF-8"));
                        ObjectName name = new ObjectName(nameBuff.toString());
                        if (mbeanServer != null && mbeanServer.isRegistered(name)) {
                            MBeanInfo info = mbeanServer.getMBeanInfo(name);
                            MBeanAttributeInfo[] attributes = info.getAttributes();
                            CustomChainingUserRegistrySynchronizer.logger.debug(id + " attributes:");
                            for (MBeanAttributeInfo attribute : attributes) {
                                Object value = mbeanServer.getAttribute(name, attribute.getName());
                                CustomChainingUserRegistrySynchronizer.logger
                                        .debug(attribute.getName() + " = " + value);
                            }
                        }
                    } catch (UnsupportedEncodingException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (MalformedObjectNameException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (InstanceNotFoundException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (IntrospectionException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (AttributeNotFoundException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (ReflectionException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (MBeanException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    } catch (IOException e) {
                        if (CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                            CustomChainingUserRegistrySynchronizer.logger.warn("Exception during logging", e);
                        }
                    }
                } // end of debug dump of active JMX bean
                if (CustomChainingUserRegistrySynchronizer.logger.isInfoEnabled()) {
                    CustomChainingUserRegistrySynchronizer.logger
                            .info("Synchronizing users and groups with user registry '" + id + "'");
                }
                if (isFullSync && CustomChainingUserRegistrySynchronizer.logger.isWarnEnabled()) {
                    CustomChainingUserRegistrySynchronizer.logger
                            .warn("Full synchronization with user registry '" + id + "'");
                    if (allowDeletions) {
                        CustomChainingUserRegistrySynchronizer.logger.warn(
                                "Some users and groups previously created by synchronization with this user registry may be removed.");
                    } else {
                        CustomChainingUserRegistrySynchronizer.logger.warn(
                                "Deletions are disabled. Users and groups removed from this registry will be logged only and will remain in the repository. Users previously found in a different registry will be moved in the repository rather than recreated.");
                    }
                }
                // Work out whether we should do the work in a separate transaction (it's most performant if we
                // bunch it into small transactions, but if we are doing a sync on login, it has to be the same
                // transaction)
                boolean requiresNew = splitTxns
                        || AlfrescoTransactionSupport.getTransactionReadState() == TxnReadState.TXN_READ_ONLY;

                try {
                    /**
                     * Do the sync with the specified plugin
                     */
                    syncWithPlugin(id, plugin, forceUpdate, isFullSync, requiresNew, visitedZoneIds,
                            allZoneIds);

                    this.applicationEventPublisher.publishEvent(new SynchronizeDirectoryEndEvent(this, id));
                } catch (final RuntimeException e) {
                    notifySyncDirectoryEnd(id, e);
                    throw e;
                }
            } // if plugin exists
        } // for each instanceId

        //End of successful synchronization here
        notifySyncEnd();
    } catch (final RuntimeException e) {
        notifySyncEnd(e);
        CustomChainingUserRegistrySynchronizer.logger.error("Synchronization aborted due to error", e);
        throw e;
    } finally {
        // Release the lock if necessary
        if (lockToken != null) {
            // Cancel the lock refresher
            // Because we may hit a perfect storm when trying to interrupt workers in their unsynchronized getTask()
            // method we can't wait indefinitely and may have to retry the shutdown
            int trys = 0;
            do {
                lockRefresher.shutdown();
                try {
                    lockRefresher.awaitTermination(CustomChainingUserRegistrySynchronizer.LOCK_TTL,
                            TimeUnit.MILLISECONDS);
                } catch (InterruptedException e) {
                }
            } while (!lockRefresher.isTerminated() && trys++ < 3);
            if (!lockRefresher.isTerminated()) {
                lockRefresher.shutdownNow();
                CustomChainingUserRegistrySynchronizer.logger.error("Failed to shut down lock refresher");
            }

            final String token = lockToken;
            this.transactionService.getRetryingTransactionHelper()
                    .doInTransaction(new RetryingTransactionCallback<Object>() {
                        public Object execute() throws Throwable {
                            CustomChainingUserRegistrySynchronizer.this.jobLockService.releaseLock(token,
                                    CustomChainingUserRegistrySynchronizer.LOCK_QNAME);
                            return null;
                        }
                    }, false, splitTxns);
        }
    }
}