Java tutorial
/* * RHQ Management Platform * Copyright (C) 2011-2014 Red Hat, Inc. * All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ package org.rhq.enterprise.server.plugins.drift.mongodb; import static org.rhq.enterprise.server.util.LookupUtil.getAgentManager; import static org.rhq.enterprise.server.util.LookupUtil.getResourceManager; import static org.rhq.enterprise.server.util.LookupUtil.getSubjectManager; import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import com.google.code.morphia.Datastore; import com.google.code.morphia.Morphia; import com.google.code.morphia.query.Query; import com.mongodb.DBObject; import com.mongodb.Mongo; import com.mongodb.WriteConcern; import com.mongodb.gridfs.GridFSDBFile; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.bson.types.ObjectId; import org.rhq.common.drift.ChangeSetReader; import org.rhq.common.drift.ChangeSetReaderImpl; import org.rhq.common.drift.FileEntry; import org.rhq.common.drift.Headers; import org.rhq.core.clientapi.agent.drift.DriftAgentService; import org.rhq.core.domain.auth.Subject; import org.rhq.core.domain.criteria.DriftChangeSetCriteria; import org.rhq.core.domain.criteria.DriftCriteria; import org.rhq.core.domain.criteria.ResourceCriteria; import org.rhq.core.domain.drift.Drift; import org.rhq.core.domain.drift.DriftChangeSet; import org.rhq.core.domain.drift.DriftChangeSetCategory; import org.rhq.core.domain.drift.DriftComposite; import org.rhq.core.domain.drift.DriftConfigurationDefinition.DriftHandlingMode; import org.rhq.core.domain.drift.DriftDefinition; import org.rhq.core.domain.drift.DriftFile; import org.rhq.core.domain.drift.DriftFileStatus; import org.rhq.core.domain.drift.dto.DriftChangeSetDTO; import org.rhq.core.domain.drift.dto.DriftDTO; import org.rhq.core.domain.drift.dto.DriftFileDTO; import org.rhq.core.domain.resource.Resource; import org.rhq.core.domain.util.PageList; import org.rhq.core.util.ZipUtil; import org.rhq.core.util.file.FileUtil; import org.rhq.core.util.stream.StreamUtil; import org.rhq.enterprise.server.agentclient.AgentClient; import org.rhq.enterprise.server.drift.DriftManagerLocal; import org.rhq.enterprise.server.plugin.pc.ServerPluginComponent; import org.rhq.enterprise.server.plugin.pc.ServerPluginContext; import org.rhq.enterprise.server.plugin.pc.drift.DriftChangeSetSummary; import org.rhq.enterprise.server.plugin.pc.drift.DriftServerPluginFacet; import org.rhq.enterprise.server.plugins.drift.mongodb.dao.ChangeSetDAO; import org.rhq.enterprise.server.plugins.drift.mongodb.dao.FileDAO; import org.rhq.enterprise.server.plugins.drift.mongodb.entities.MongoDBChangeSet; import org.rhq.enterprise.server.plugins.drift.mongodb.entities.MongoDBChangeSetEntry; import org.rhq.enterprise.server.plugins.drift.mongodb.entities.MongoDBFile; import org.rhq.enterprise.server.resource.ResourceManagerLocal; import org.rhq.enterprise.server.util.CriteriaQuery; import org.rhq.enterprise.server.util.CriteriaQueryExecutor; import org.rhq.enterprise.server.util.LookupUtil; public class MongoDBDriftServer implements DriftServerPluginFacet, ServerPluginComponent { private Log log = LogFactory.getLog(MongoDBDriftServer.class); private Mongo connection; private Morphia morphia; private Datastore ds; private ChangeSetDAO changeSetDAO; private FileDAO fileDAO; public Mongo getConnection() { return connection; } public void setConnection(Mongo connection) { this.connection = connection; } public Morphia getMorphia() { return morphia; } public void setMorphia(Morphia morphia) { this.morphia = morphia; } public Datastore getDatastore() { return ds; } public void setDatastore(Datastore ds) { this.ds = ds; } public ChangeSetDAO getChangeSetDAO() { return changeSetDAO; } public void setChangeSetDAO(ChangeSetDAO changeSetDAO) { this.changeSetDAO = changeSetDAO; } public FileDAO getFileDAO() { return fileDAO; } public void setFileDAO(FileDAO fileDAO) { this.fileDAO = fileDAO; } @Override public void initialize(ServerPluginContext context) throws Exception { connection = new Mongo("127.0.0.1"); morphia = new Morphia().map(MongoDBChangeSet.class).map(MongoDBChangeSetEntry.class).map(MongoDBFile.class); ds = morphia.createDatastore(connection, "rhq"); changeSetDAO = new ChangeSetDAO(morphia, connection, "rhq"); fileDAO = new FileDAO(ds.getDB()); // Note that a write concern of SAFE does not cause the write operation // to wait for the server to flush to disk which occurrs every one // minute by default. With journaled enabled however (which is the // default), we get the ability to recover from errors such as failure // to flush to disk. If we absolutely need to wait and ensure that the // write is successfully written to disk, we could use a write concern // of FSYNC_SAFE; however, this should be used very judiciously as it // will introduce a significant performance overhead on the server as // it forces a flush to disk on every write (from this connect). ds.setDefaultWriteConcern(WriteConcern.SAFE); } @Override public void start() { } @Override public void stop() { } @Override public void shutdown() { } @Override public DriftChangeSetSummary saveChangeSet(final Subject subject, final int resourceId, final File changeSetZip) throws Exception { final DriftChangeSetSummary summary = new DriftChangeSetSummary(); ZipUtil.walkZipFile(changeSetZip, new ZipUtil.ZipEntryVisitor() { @Override public boolean visit(ZipEntry zipEntry, ZipInputStream stream) throws Exception { ChangeSetReader reader = new ChangeSetReaderImpl(new BufferedReader(new InputStreamReader(stream))); Headers headers = reader.getHeaders(); List<DriftFileDTO> missingContent = new LinkedList<DriftFileDTO>(); MongoDBChangeSet changeSet = new MongoDBChangeSet(); changeSet.setCategory(headers.getType()); changeSet.setResourceId(resourceId); changeSet.setDriftDefinitionId(headers.getDriftDefinitionId()); changeSet.setDriftDefinitionName(headers.getDriftDefinitionName()); changeSet.setDriftHandlingMode(DriftHandlingMode.normal); changeSet.setVersion(headers.getVersion()); summary.setCategory(headers.getType()); summary.setResourceId(resourceId); summary.setDriftDefinitionName(headers.getDriftDefinitionName()); summary.setCreatedTime(changeSet.getCtime()); for (FileEntry fileEntry : reader) { String path = FileUtil.useForwardSlash(fileEntry.getFile()); MongoDBChangeSetEntry entry = new MongoDBChangeSetEntry(path, fileEntry.getType()); switch (fileEntry.getType()) { case FILE_ADDED: entry.setNewFileHash(fileEntry.getNewSHA()); if (fileDAO.findOne(fileEntry.getNewSHA()) == null) { missingContent.add(newDriftFile(fileEntry.getNewSHA())); } break; case FILE_CHANGED: entry.setOldFileHash(fileEntry.getOldSHA()); entry.setNewFileHash(fileEntry.getNewSHA()); if (fileDAO.findOne(fileEntry.getNewSHA()) == null) { missingContent.add(newDriftFile(fileEntry.getNewSHA())); } if (fileDAO.findOne(fileEntry.getOldSHA()) == null) { missingContent.add(newDriftFile(fileEntry.getNewSHA())); } break; default: // FILE_REMOVED entry.setOldFileHash(fileEntry.getOldSHA()); if (fileDAO.findOne(fileEntry.getOldSHA()) == null) { missingContent.add(newDriftFile(fileEntry.getOldSHA())); } } changeSet.add(entry); // we are taking advantage of the fact that we know the summary is only used by the server // if the change set is a DRIFT report. If its a coverage report, it is not used (we do // not alert on coverage reports) - so don't waste memory by collecting all the paths // when we know they aren't going to be used anyway. if (headers.getType() == DriftChangeSetCategory.DRIFT) { summary.addDriftPathname(path); } } ds.save(changeSet); // The following section of code really should not be part of the plugin // implementation, but it is currently required due to a flaw in the design // of the drift plugin framework. Two things are done here. First, if we // successfully persist the change set, then we need to send an // acknowledgement to the agent. This is critical because the agent will // in effect suspend drift detection (for the particular definition) until // it receives the ACK. Secondly, we need to tell the agent to send the // actual file bits for any change set content we do not have. DriftAgentService driftService = getDriftAgentService(resourceId); driftService.ackChangeSet(headers.getResourceId(), headers.getDriftDefinitionName()); if (!missingContent.isEmpty()) { driftService.requestDriftFiles(resourceId, headers, missingContent); } return true; } }); return summary; } protected DriftAgentService getDriftAgentService(int resourceId) { AgentClient agent = getAgentManager().getAgentClient(getSubjectManager().getOverlord(), resourceId); return agent.getDriftAgentService(); } private DriftFileDTO newDriftFile(String hash) { DriftFileDTO file = new DriftFileDTO(); file.setHashId(hash); return file; } private DriftFileDTO newDriftFile(String hash, DriftFileStatus status) { DriftFileDTO file = new DriftFileDTO(); file.setHashId(hash); file.setStatus(status); return file; } @Override public void saveChangeSetFiles(final Subject subject, final File changeSetFilesZip) throws Exception { String zipFileName = changeSetFilesZip.getName(); File tmpDir = new File(System.getProperty("java.io.tmpdir")); File dir = FileUtil.createTempDirectory(zipFileName.substring(0, zipFileName.indexOf(".")), null, tmpDir); dir.mkdir(); ZipUtil.unzipFile(changeSetFilesZip, dir); for (File file : dir.listFiles()) { fileDAO.save(file); file.delete(); } } @Override public PageList<? extends DriftChangeSet<?>> findDriftChangeSetsByCriteria(Subject subject, DriftChangeSetCriteria criteria) { Mapper mapper = new Mapper(); List<MongoDBChangeSet> changeSets = changeSetDAO.findByChangeSetCritiera(criteria); PageList<DriftChangeSetDTO> results = new PageList<DriftChangeSetDTO>(); for (MongoDBChangeSet changeSet : changeSets) { DriftChangeSetDTO changeSetDTO = mapper.toDTO(changeSet); if (criteria.isFetchDrifts()) { Set<DriftDTO> entries = new HashSet<DriftDTO>(); for (MongoDBChangeSetEntry entry : changeSet.getDrifts()) { DriftDTO driftDTO = mapper.toDTO(entry); driftDTO.setChangeSet(changeSetDTO); entries.add(driftDTO); } changeSetDTO.setDrifts(entries); } results.add(changeSetDTO); } return results; } @Override public PageList<? extends Drift<?, ?>> findDriftsByCriteria(Subject subject, DriftCriteria criteria) { Mapper mapper = new Mapper(); List<MongoDBChangeSetEntry> entries = changeSetDAO.findEntries(criteria); PageList<DriftDTO> results = new PageList<DriftDTO>(); for (MongoDBChangeSetEntry entry : entries) { DriftDTO driftDTO = mapper.toDTO(entry); if (criteria.isFetchChangeSet()) { driftDTO.setChangeSet(mapper.toDTO(entry.getChangeSet())); } results.add(driftDTO); } return results; } @Override public PageList<DriftComposite> findDriftCompositesByCriteria(Subject subject, DriftCriteria criteria) { Mapper mapper = new Mapper(); List<MongoDBChangeSetEntry> entries = changeSetDAO.findEntries(criteria); Map<Integer, Resource> resources = loadResourceMap(subject, criteria.getFilterResourceIds()); PageList<DriftComposite> results = new PageList<DriftComposite>(); for (MongoDBChangeSetEntry entry : entries) { MongoDBChangeSet changeSet = entry.getChangeSet(); DriftDTO driftDTO = mapper.toDTO(entry); if (criteria.isFetchChangeSet()) { DriftChangeSetDTO changeSetDTO = mapper.toDTO(changeSet); driftDTO.setChangeSet(changeSetDTO); } results.add(new DriftComposite(driftDTO, resources.get(changeSet.getResourceId()), changeSet.getDriftDefinitionName())); } return results; } @Override public DriftFile getDriftFile(Subject subject, String hashId) throws Exception { GridFSDBFile gridFSDBFile = fileDAO.findById(hashId); if (gridFSDBFile == null) { return null; } return newDriftFile(hashId, DriftFileStatus.LOADED); } @Override public void purgeByDriftDefinitionName(Subject subject, int resourceId, String driftDefName) throws Exception { changeSetDAO.deleteChangeSets(resourceId, driftDefName); } @Override public int purgeOrphanedDriftFiles(Subject subject, long purgeMillis) { // TODO implement me! return 0; } public void purgeOrphanedContent() { for (DBObject object : fileDAO.getFileListCursor()) { String id = (String) object.get("_id"); Query<MongoDBChangeSet> query = ds.createQuery(MongoDBChangeSet.class); query.or(query.criteria("files.newFileHash").equal(id), query.criteria("files.oldFileHash").equal(id)); query.limit(1); if (query.get() == null) { fileDAO.delete(id); } } } @Override public String getDriftFileBits(Subject subject, String hash) { GridFSDBFile file = fileDAO.findById(hash); if (file == null) { return null; } return new String(StreamUtil.slurp(file.getInputStream())); } Map<Integer, Resource> loadResourceMap(final Subject subject, Integer[] resourceIds) { ResourceCriteria criteria = new ResourceCriteria(); criteria.addFilterIds(resourceIds); final ResourceManagerLocal resourceMgr = getResourceManager(); //Use CriteriaQuery to automatically chunk/page through criteria query results CriteriaQueryExecutor<Resource, ResourceCriteria> queryExecutor = new CriteriaQueryExecutor<Resource, ResourceCriteria>() { @Override public PageList<Resource> execute(ResourceCriteria criteria) { return resourceMgr.findResourcesByCriteria(subject, criteria); } }; CriteriaQuery<Resource, ResourceCriteria> resources = new CriteriaQuery<Resource, ResourceCriteria>( criteria, queryExecutor); Map<Integer, Resource> map = new HashMap<Integer, Resource>(); for (Resource r : resources) { map.put(r.getId(), r); } return map; } public String persistChangeSet(Subject subject, DriftChangeSet<?> changeSet) { // if this is a resource level change set we need to fetch the definition so that // we can persist the definition name; otherwise, we will not be able to delete // this and any other future change sets that belong to the definition. // TODO Refactor caller logic of persistChangeSet to pass the definition name // It might make sense to break persistChangeSet into two separate methods, // something along the lines of persistTemplateChangeSet and // persistDefinitionChangeSet where the definition is passed as an argument to // the latter. As it stands right now, each plugin implementation will have // duplicate logic for determining whether this is a template or definition change // set. That logic could be pulled up into DriftManagerBean. MongoDBChangeSet newChangeSet = new MongoDBChangeSet(); newChangeSet.setResourceId(changeSet.getResourceId()); newChangeSet.setDriftDefinitionId(changeSet.getDriftDefinitionId()); newChangeSet.setDriftHandlingMode(changeSet.getDriftHandlingMode()); newChangeSet.setCategory(changeSet.getCategory()); newChangeSet.setDriftDefinitionId(changeSet.getDriftDefinitionId()); if (!isTemplateChangeSet(changeSet)) { DriftDefinition driftDef = getDriftDef(subject, changeSet.getDriftDefinitionId()); if (driftDef == null) { throw new IllegalArgumentException( "Cannot persist change set. " + DriftDefinition.class.getSimpleName() + " with id " + changeSet.getDriftDefinitionId() + " cannot be found."); } newChangeSet.setDriftDefinitionName(driftDef.getName()); } for (Drift drift : changeSet.getDrifts()) { MongoDBChangeSetEntry entry = new MongoDBChangeSetEntry(); entry.setPath(drift.getPath()); entry.setCategory(drift.getCategory()); // we only need to initialize the newFileHash property here since each drift // is going to be a FILE_ADDED entry. entry.setNewFileHash(drift.getNewDriftFile().getHashId()); newChangeSet.add(entry); } changeSetDAO.save(newChangeSet); return newChangeSet.getId(); } private boolean isTemplateChangeSet(DriftChangeSet<?> changeSet) { return changeSet.getResourceId() == 0 && changeSet.getDriftDefinitionId() == 0; } @Override public String copyChangeSet(Subject subject, String changeSetId, int driftDefId, int resourceId) { // TODO Pass definition name in as an argument // We need to store the definition name to support deletes. This method is invoked // from DriftManagerBean.updateDriftDefinition when creating a definition from a // pinned template. Because the transaction in which the definition is created has // not yet been committed when this method is invoked, we cannot look up the // definition. MongoDBChangeSet changeSet = changeSetDAO.findOne("id", new ObjectId(changeSetId)); changeSet.setDriftDefinitionId(driftDefId); changeSet.setResourceId(resourceId); changeSet.setId(new ObjectId()); changeSetDAO.save(changeSet); return changeSet.getId(); } @Override public byte[] getDriftFileAsByteArray(Subject subject, String hash) { GridFSDBFile file = fileDAO.findById(hash); if (file == null) { return null; } return StreamUtil.slurp(file.getInputStream()); } protected DriftDefinition getDriftDef(Subject subject, int id) { DriftManagerLocal driftMgr = LookupUtil.getDriftManager(); return driftMgr.getDriftDefinition(subject, id); } }