Java tutorial
/************************************************************************* * (c) Copyright 2017 Hewlett Packard Enterprise Development Company LP * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; version 3 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/. * * This file may incorporate work covered under the following copyright * and permission notice: * * Software License Agreement (BSD License) * * Copyright (c) 2008, Regents of the University of California * All rights reserved. * * Redistribution and use of this software in source and binary forms, * with or without modification, are permitted provided that the * following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. USERS OF THIS SOFTWARE ACKNOWLEDGE * THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE LICENSED MATERIAL, * COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS SOFTWARE, * AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING * IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, * SANTA BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, * WHICH IN THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, * REPLACEMENT OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO * IDENTIFIED, OR WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT * NEEDED TO COMPLY WITH ANY SUCH LICENSES OR RIGHTS. ************************************************************************/ package com.eucalyptus.blockstorage.async; import java.util.concurrent.ConcurrentHashMap; import java.util.Date; import java.util.List; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.hibernate.Criteria; import org.hibernate.criterion.Example; import org.hibernate.criterion.MatchMode; import org.hibernate.criterion.Restrictions; import com.eucalyptus.blockstorage.LogicalStorageManager; import com.eucalyptus.blockstorage.S3SnapshotTransfer; import com.eucalyptus.blockstorage.entities.SnapshotInfo; import com.eucalyptus.blockstorage.entities.SnapshotInfo_; import com.eucalyptus.blockstorage.util.StorageProperties; import com.eucalyptus.entities.Entities; import com.eucalyptus.entities.TransactionResource; import com.eucalyptus.entities.Transactions; import com.eucalyptus.storage.common.CheckerTask; import com.eucalyptus.util.EucalyptusCloudException; import com.eucalyptus.util.metrics.MonitoredAction; import com.eucalyptus.util.metrics.ThruputMetrics; import edu.ucsb.eucalyptus.util.EucaSemaphore; import edu.ucsb.eucalyptus.util.EucaSemaphoreDirectory; /** * Checker task for removing snapshots marked in deleting status * * @author Swathi Gangisetty * */ public class SnapshotDeleter extends CheckerTask { private static Logger LOG = Logger.getLogger(SnapshotDeleter.class); private static ConcurrentHashMap<String, Date> retryMap = new ConcurrentHashMap<String, Date>(); // Retry for two days max // TODO make this configurable? private static final long RETRY_MAX_MILLIS = (2/*days*/ * 24 * 60 * 60 * 1000)/*millisecs*/; private static final long RETRY_MAX_MINUTES = RETRY_MAX_MILLIS / 1000 / 60; private static final long RETRY_MAX_HOURS = RETRY_MAX_MINUTES / 60; private LogicalStorageManager blockManager; private S3SnapshotTransfer snapshotTransfer; public SnapshotDeleter(LogicalStorageManager blockManager) { this.name = SnapshotDeleter.class.getSimpleName(); this.runInterval = 30; // runs every 30 seconds, TODO make this configurable? this.blockManager = blockManager; } public SnapshotDeleter(LogicalStorageManager blockManager, S3SnapshotTransfer mock) { this(blockManager); this.snapshotTransfer = mock; } @Override public void run() { // Clean up on EBS backend deleteFromEBS(); // Clean up on OSG deleteFromOSG(); } private void deleteFromEBS() { try { SnapshotInfo searchSnap = new SnapshotInfo(); searchSnap.setStatus(StorageProperties.Status.deleting.toString()); List<SnapshotInfo> snapshotsToBeDeleted = null; try { snapshotsToBeDeleted = Transactions.findAll(searchSnap); } catch (Exception e) { LOG.warn("Failed to lookup snapshots marked for deletion", e); return; } if (snapshotsToBeDeleted != null && !snapshotsToBeDeleted.isEmpty()) { LOG.trace("Deleting snapshots from EBS"); for (SnapshotInfo snap : snapshotsToBeDeleted) { try { String snapshotId = snap.getSnapshotId(); LOG.debug("Snapshot " + snapshotId + " was marked for deletion from EBS backend. Evaluating prerequisites for cleanup..."); if (snap.getIsOrigin() != null && snap.getIsOrigin()) { // check if snapshot originates in this az // acquire semaphore before deleting to avoid concurrent interaction with delta creation process LOG.debug("Snapshot " + snapshotId + " originates from this az, acquire semaphore before deletion"); EucaSemaphore snapSemaphore = EucaSemaphoreDirectory.getSolitarySemaphore(snapshotId); try { try { snapSemaphore.acquire(); } catch (InterruptedException ex) { LOG.warn("Cannot process deletion of " + snapshotId + " due to an error acquiring semaphore. Will retry again later"); continue; } deleteSnapFromEBS(snap); } finally { snapSemaphore.release(); EucaSemaphoreDirectory.removeSemaphore(snapshotId); } } else { // either pre 4.4 snapshot or snapshot does not originate in this az // no need to acquire semaphore, delete straight away deleteSnapFromEBS(snap); } } catch (Exception e) { LOG.warn("Failed to process deletion for " + snap.getSnapshotId() + " on EBS backend", e); continue; } finally { ThruputMetrics.endOperation(MonitoredAction.DELETE_SNAPSHOT, snap.getSnapshotId(), System.currentTimeMillis()); } } } else { LOG.trace("No snapshots marked for deletion"); } } catch (Exception e) { // could catch InterruptedException LOG.warn("Unable to remove snapshots marked for deletion from EBS backend", e); return; } } private void deleteFromOSG() { try { // Get the snapshots that are deleted from EBS but not yet deleted from OSG, // in reverse time order so we never try to delete a parent before a child List<SnapshotInfo> snapshotsToBeDeleted = null; try (TransactionResource tr = Entities.transactionFor(SnapshotInfo.class)) { snapshotsToBeDeleted = Entities.criteriaQuery(Entities.restriction(SnapshotInfo.class) .equal(SnapshotInfo_.status, StorageProperties.Status.deletedfromebs.toString()).build()) .orderByDesc(SnapshotInfo_.startTime).list(); tr.commit(); } catch (Exception e) { LOG.warn("Failed database lookup of snapshots marked for deletion from OSG", e); return; } if (snapshotsToBeDeleted != null && !snapshotsToBeDeleted.isEmpty()) { LOG.trace("Deleting snapshots from OSG"); for (SnapshotInfo snap : snapshotsToBeDeleted) { try { String snapshotId = snap.getSnapshotId(); LOG.debug("Snapshot " + snapshotId + " was marked for deletion from OSG. Evaluating prerequisites for cleanup..."); if (snap.getIsOrigin() == null) { // old snapshot prior to 4.4 LOG.debug("Snapshot " + snapshotId + " may have been created prior to incremental snapshot support"); deleteSnapFromOSG(snap); // delete snapshot } else if (snap.getIsOrigin()) { // snapshot originated in the same az LOG.debug("Snapshot " + snapshotId + " originates from this az, verifying if it's needed to restore other snapshots"); List<SnapshotInfo> nextSnaps = null; try (TransactionResource tr = Entities.transactionFor(SnapshotInfo.class)) { SnapshotInfo nextSnapSearch = new SnapshotInfo(); nextSnapSearch.setScName(snap.getScName()); nextSnapSearch.setVolumeId(snap.getVolumeId()); nextSnapSearch.setIsOrigin(Boolean.TRUE); nextSnapSearch.setPreviousSnapshotId(snap.getSnapshotId()); Criteria search = Entities.createCriteria(SnapshotInfo.class); search.add(Example.create(nextSnapSearch).enableLike(MatchMode.EXACT)); search.add(StorageProperties.SNAPSHOT_DELTA_RESTORATION_CRITERION); search.setReadOnly(true); nextSnaps = (List<SnapshotInfo>) search.list(); tr.commit(); } catch (Exception e) { LOG.warn("Failed to lookup snapshots that may depend on " + snapshotId + " for reconstruction", e); return; } if (nextSnaps != null && !nextSnaps.isEmpty()) { // Found deltas that might depend on this snapshot for reconstruction, don't delete. // Normally there will be only 1 next snap, optimize for that case. String nextSnapIds = nextSnaps.get(0).getSnapshotId(); if (nextSnaps.size() > 1) { for (int nextSnapIdNum = 1; nextSnapIdNum < nextSnaps.size(); nextSnapIdNum++) { nextSnapIds = nextSnapIds + ", " + nextSnaps.get(nextSnapIdNum).getSnapshotId(); } } LOG.debug("Snapshot " + snapshotId + " is required for restoring other snapshots in the system." + " Cannot delete from OSG. Direct children of this snapshot: " + nextSnapIds); } else { LOG.debug("Snapshot " + snapshotId + " is not required for restoring other snapshots in the system"); deleteSnapFromOSG(snap); // delete snapshot } } else { // snapshot originated in a different az // skip evaluation and just mark the snapshot deleted, let the source az deal with the osg remnants TODO fix this later LOG.debug("Snapshot " + snapshotId + " orignated from a different az, let the source az deal with deletion from OSG"); markSnapDeleted(snapshotId); } } catch (Exception e) { LOG.warn("Failed to process deletion for " + snap.getSnapshotId() + " on ObjectStorageGateway", e); continue; } } } else { LOG.trace("No snapshots marked for deletion from OSG"); } } catch (Exception e) { // could catch InterruptedException LOG.warn("Unable to remove snapshots marked for deletion from OSG", e); return; } } private void deleteSnapFromEBS(SnapshotInfo snap) { String snapshotId = snap.getSnapshotId(); LOG.debug("Deleting snapshot " + snapshotId + " from EBS backend..."); try { blockManager.deleteSnapshot(snapshotId, snap.getSnapPointId()); } catch (EucalyptusCloudException e) { LOG.warn("Unable to delete " + snapshotId + " from EBS backend. Will retry later", e); return; } if (StringUtils.isNotBlank(snap.getSnapshotLocation())) { // snapshot removal from s3 needs evaluation markSnapDeletedFromEBS(snapshotId); LOG.debug("Snapshot " + snapshotId + " set to 'deletedfromebs' state from EBS cleanup"); } else { // no evidence of snapshot upload to OSG, mark the snapshot as deleted markSnapDeleted(snapshotId); LOG.debug("Snapshot " + snapshotId + " set to 'deleted' state from EBS cleanup"); } } private void deleteSnapFromOSG(SnapshotInfo snap) { if (StringUtils.isNotBlank(snap.getSnapshotLocation())) { LOG.debug("Deleting snapshot " + snap.getSnapshotId() + " from ObjectStorageGateway"); try { String[] names = SnapshotInfo.getSnapshotBucketKeyNames(snap.getSnapshotLocation()); if (snapshotTransfer == null) { snapshotTransfer = new S3SnapshotTransfer(); } snapshotTransfer.setSnapshotId(snap.getSnapshotId()); snapshotTransfer.setBucketName(names[0]); snapshotTransfer.setKeyName(names[1]); snapshotTransfer.delete(); markSnapDeleted(snap.getSnapshotId()); } catch (Exception e) { LOG.info("Failed to delete snapshot " + snap.getSnapshotId() + " from ObjectStorageGateway.", e); retryOrDelete(snap.getSnapshotId(), /*fromEBS*/ false, "from OSG"); return; } } else { LOG.debug("Snapshot location missing for " + snap.getSnapshotId() + ". It may be created later. Skipping deletion from ObjectStorageGateway for now."); retryOrDelete(snap.getSnapshotId(), /*fromEBS*/ false, "from OSG"); } } private void markSnapDeleted(String snapshotId) { try (TransactionResource tran = Entities.transactionFor(SnapshotInfo.class)) { SnapshotInfo foundSnapshotInfo = Entities.uniqueResult(new SnapshotInfo(snapshotId)); foundSnapshotInfo.setStatus(StorageProperties.Status.deleted.toString()); foundSnapshotInfo.setDeletionTime(new Date()); tran.commit(); // Only remove it from the retry list (OK if it's not there) if we // successfully set the snap to deleted. Otherwise it may be put back // on the list on the next run and start a retry period again. retryMap.remove(snapshotId); LOG.debug("Snapshot " + snapshotId + " set to 'deleted' state"); } catch (Exception e) { LOG.warn("Failed to update status for " + snapshotId + " to deleted", e); } } private void markSnapDeletedFromEBS(String snapshotId) { try (TransactionResource tran = Entities.transactionFor(SnapshotInfo.class)) { SnapshotInfo foundSnapshotInfo = Entities.uniqueResult(new SnapshotInfo(snapshotId)); foundSnapshotInfo.setStatus(StorageProperties.Status.deletedfromebs.toString()); tran.commit(); } catch (Exception e) { LOG.warn("Failed to update status for " + snapshotId + " to deletedfromebs", e); } } private boolean retryOrDelete(String snapshotId, boolean fromEBS, String fromEBSorOSG) { Date retryStartDate = null; Date now = new Date(); boolean timedOut = false; if ((retryStartDate = retryMap.get(snapshotId)) == null) { LOG.warn("Deleting snapshot " + snapshotId + " " + fromEBSorOSG + " failed. Will retry until " + new Date(now.getTime() + RETRY_MAX_MILLIS)); retryMap.put(snapshotId, now); } else { long retryStartMillis = retryStartDate.getTime(); if (now.getTime() - retryStartMillis > RETRY_MAX_MILLIS) { timedOut = true; // Compiler warns of "dead code" here due to constants. // Kept here so we can change RETRY_MAX_MILLIS for test purposes. String retryMaxHumanReadable = (RETRY_MAX_HOURS < 1 ? (RETRY_MAX_MINUTES + " minutes") : (RETRY_MAX_HOURS + " hours")); LOG.error("Deleting snapshot " + snapshotId + " " + fromEBSorOSG + " failed on its final attempt, after " + retryMaxHumanReadable + ". Giving up trying to delete it " + fromEBSorOSG); try { if (fromEBS) { markSnapDeletedFromEBS(snapshotId); } else { markSnapDeleted(snapshotId); } } catch (Exception e2) { LOG.info("Snapshot " + snapshotId + " could not be deleted " + fromEBSorOSG + ". It may have already been deleted in another thread."); } } else { LOG.warn("Deleting snapshot " + snapshotId + " " + fromEBSorOSG + " failed again. Will retry until " + new Date(retryStartMillis + RETRY_MAX_MILLIS)); } } return timedOut; } }