Java tutorial
/** * Licensed to JumpMind Inc under one or more contributor * license agreements. See the NOTICE file distributed * with this work for additional information regarding * copyright ownership. JumpMind Inc licenses this file * to you under the GNU General Public License, version 3.0 (GPLv3) * (the "License"); you may not use this file except in compliance * with the License. * * You should have received a copy of the GNU General Public License, * version 3.0 (GPLv3) along with this library; if not, see * <http://www.gnu.org/licenses/>. * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.jumpmind.symmetric.service.impl; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.sql.Types; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.lang.StringUtils; import org.jumpmind.db.model.Table; import org.jumpmind.db.sql.ISqlReadCursor; import org.jumpmind.db.sql.ISqlRowMapper; import org.jumpmind.db.sql.ISqlTransaction; import org.jumpmind.db.sql.Row; import org.jumpmind.exception.IoException; import org.jumpmind.symmetric.ISymmetricEngine; import org.jumpmind.symmetric.SymmetricException; import org.jumpmind.symmetric.common.Constants; import org.jumpmind.symmetric.common.ParameterConstants; import org.jumpmind.symmetric.common.TableConstants; import org.jumpmind.symmetric.file.DirectorySnapshot; import org.jumpmind.symmetric.file.FileConflictException; import org.jumpmind.symmetric.file.FileSyncZipDataWriter; import org.jumpmind.symmetric.file.FileTriggerTracker; import org.jumpmind.symmetric.io.data.CsvData; import org.jumpmind.symmetric.io.data.DataEventType; import org.jumpmind.symmetric.io.stage.IStagedResource; import org.jumpmind.symmetric.io.stage.IStagingManager; import org.jumpmind.symmetric.model.BatchAck; import org.jumpmind.symmetric.model.Channel; import org.jumpmind.symmetric.model.Data; import org.jumpmind.symmetric.model.FileConflictStrategy; import org.jumpmind.symmetric.model.FileSnapshot; import org.jumpmind.symmetric.model.FileSnapshot.LastEventType; import org.jumpmind.symmetric.model.FileTrigger; import org.jumpmind.symmetric.model.FileTriggerRouter; import org.jumpmind.symmetric.model.IncomingBatch; import org.jumpmind.symmetric.model.Node; import org.jumpmind.symmetric.model.NodeCommunication; import org.jumpmind.symmetric.model.NodeCommunication.CommunicationType; import org.jumpmind.symmetric.model.NodeSecurity; import org.jumpmind.symmetric.model.OutgoingBatch; import org.jumpmind.symmetric.model.OutgoingBatch.Status; import org.jumpmind.symmetric.model.OutgoingBatches; import org.jumpmind.symmetric.model.ProcessInfo; import org.jumpmind.symmetric.model.ProcessInfoKey; import org.jumpmind.symmetric.model.ProcessType; import org.jumpmind.symmetric.model.RemoteNodeStatus; import org.jumpmind.symmetric.model.RemoteNodeStatuses; import org.jumpmind.symmetric.service.ClusterConstants; import org.jumpmind.symmetric.service.IFileSyncService; import org.jumpmind.symmetric.service.IIncomingBatchService; import org.jumpmind.symmetric.service.INodeCommunicationService; import org.jumpmind.symmetric.service.INodeCommunicationService.INodeCommunicationExecutor; import org.jumpmind.symmetric.service.INodeService; import org.jumpmind.symmetric.transport.IIncomingTransport; import org.jumpmind.symmetric.transport.IOutgoingTransport; import org.jumpmind.symmetric.transport.IOutgoingWithResponseTransport; import org.jumpmind.util.AppUtils; import bsh.EvalError; import bsh.Interpreter; import bsh.TargetError; public class FileSyncService extends AbstractOfflineDetectorService implements IFileSyncService, INodeCommunicationExecutor { private ISymmetricEngine engine; // TODO cache trigger routers public FileSyncService(ISymmetricEngine engine) { super(engine.getParameterService(), engine.getSymmetricDialect(), engine.getExtensionService()); this.engine = engine; setSqlMap(new FileSyncServiceSqlMap(platform, createSqlReplacementTokens())); } public boolean refreshFromDatabase() { // TODO implement with cache return false; } public void trackChanges(boolean force) { if (force || engine.getClusterService().lock(ClusterConstants.FILE_SYNC_TRACKER)) { try { log.debug("Attempting to get exclusive lock for file sync track changes"); if (engine.getClusterService().lock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_EXCLUSIVE, getParameterService().getLong(ParameterConstants.FILE_SYNC_LOCK_WAIT_MS))) { try { log.debug("Tracking changes for file sync"); List<FileTriggerRouter> fileTriggerRouters = getFileTriggerRoutersForCurrentNode(); for (FileTriggerRouter fileTriggerRouter : fileTriggerRouters) { if (fileTriggerRouter.isEnabled()) { FileTriggerTracker tracker = new FileTriggerTracker(fileTriggerRouter, getDirectorySnapshot(fileTriggerRouter)); try { DirectorySnapshot dirSnapshot = tracker.trackChanges(); for (FileSnapshot fileSnapshot : dirSnapshot) { File file = fileTriggerRouter.getFileTrigger() .createSourceFile(fileSnapshot); String filePath = file.getParentFile().getPath().replace('\\', '/'); String fileName = file.getName(); String nodeId = findSourceNodeIdFromFileIncoming(filePath, fileName, fileSnapshot.getFileModifiedTime()); if (StringUtils.isNotBlank(nodeId)) { fileSnapshot.setLastUpdateBy(nodeId); } else { fileSnapshot.setLastUpdateBy(null); } log.debug("Captured change " + fileSnapshot.getLastEventType() + " change of " + fileSnapshot.getFileName() + " (lastmodified=" + fileSnapshot.getFileModifiedTime() + ",size=" + fileSnapshot.getFileSize() + ") from " + fileSnapshot.getLastUpdateBy()); } save(dirSnapshot); } catch (Exception ex) { log.error("Failed to track changes for file trigger router: " + fileTriggerRouter.getFileTrigger().getTriggerId() + "::" + fileTriggerRouter.getRouter().getRouterId(), ex); } } } deleteFromFileIncoming(); } finally { log.debug("Done tracking changes for file sync"); engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_EXCLUSIVE); } } else { log.warn("Did not run the track file sync changes process because it was shared locked"); } } finally { if (!force) { engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_TRACKER); } } } else { log.debug("Did not run the track file sync changes process because it was cluster locked"); } } protected String findSourceNodeIdFromFileIncoming(String filePath, String fileName, long lastUpdateDate) { return sqlTemplate.queryForString(getSql("findNodeIdFromFileIncoming"), filePath, fileName, lastUpdateDate); } protected void deleteFromFileIncoming() { sqlTemplate.update(getSql("deleteFileIncoming")); } public List<FileTrigger> getFileTriggers() { return sqlTemplate.query(getSql("selectFileTriggersSql"), new FileTriggerMapper()); } public FileTrigger getFileTrigger(String triggerId) { return sqlTemplate.queryForObject(getSql("selectFileTriggersSql", "triggerIdWhere"), new FileTriggerMapper(), triggerId); } public List<FileTriggerRouter> getFileTriggerRoutersForCurrentNode() { return sqlTemplate.query(getSql("selectFileTriggerRoutersSql", "fileTriggerRoutersForCurrentNodeWhere"), new FileTriggerRouterMapper(), parameterService.getNodeGroupId()); } public List<FileTriggerRouter> getFileTriggerRouters() { return sqlTemplate.query(getSql("selectFileTriggerRoutersSql"), new FileTriggerRouterMapper()); } public FileTriggerRouter getFileTriggerRouter(String triggerId, String routerId) { return sqlTemplate.queryForObject(getSql("selectFileTriggerRoutersSql", "whereTriggerRouterId"), new FileTriggerRouterMapper(), triggerId, routerId); } public void saveFileTrigger(FileTrigger fileTrigger) { fileTrigger.setLastUpdateTime(new Date()); if (0 == sqlTemplate.update(getSql("updateFileTriggerSql"), new Object[] { fileTrigger.getBaseDir(), fileTrigger.isRecurse() ? 1 : 0, fileTrigger.getIncludesFiles(), fileTrigger.getExcludesFiles(), fileTrigger.isSyncOnCreate() ? 1 : 0, fileTrigger.isSyncOnModified() ? 1 : 0, fileTrigger.isSyncOnDelete() ? 1 : 0, fileTrigger.isSyncOnCtlFile() ? 1 : 0, fileTrigger.isDeleteAfterSync() ? 1 : 0, fileTrigger.getBeforeCopyScript(), fileTrigger.getAfterCopyScript(), fileTrigger.getLastUpdateBy(), fileTrigger.getLastUpdateTime(), fileTrigger.getChannelId(), fileTrigger.getReloadChannelId(), fileTrigger.getTriggerId() }, new int[] { Types.VARCHAR, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR })) { fileTrigger.setCreateTime(fileTrigger.getLastUpdateTime()); sqlTemplate.update(getSql("insertFileTriggerSql"), new Object[] { fileTrigger.getBaseDir(), fileTrigger.isRecurse() ? 1 : 0, fileTrigger.getIncludesFiles(), fileTrigger.getExcludesFiles(), fileTrigger.isSyncOnCreate() ? 1 : 0, fileTrigger.isSyncOnModified() ? 1 : 0, fileTrigger.isSyncOnDelete() ? 1 : 0, fileTrigger.isSyncOnCtlFile() ? 1 : 0, fileTrigger.isDeleteAfterSync() ? 1 : 0, fileTrigger.getBeforeCopyScript(), fileTrigger.getAfterCopyScript(), fileTrigger.getLastUpdateBy(), fileTrigger.getLastUpdateTime(), fileTrigger.getTriggerId(), fileTrigger.getCreateTime(), fileTrigger.getChannelId(), fileTrigger.getReloadChannelId() }, new int[] { Types.VARCHAR, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR }); } } public void saveFileTriggerRouter(FileTriggerRouter fileTriggerRouter) { fileTriggerRouter.setLastUpdateTime(new Date()); if (0 == sqlTemplate.update(getSql("updateFileTriggerRouterSql"), new Object[] { fileTriggerRouter.isEnabled() ? 1 : 0, fileTriggerRouter.isInitialLoadEnabled() ? 1 : 0, fileTriggerRouter.getTargetBaseDir(), fileTriggerRouter.getConflictStrategy().name(), fileTriggerRouter.getLastUpdateBy(), fileTriggerRouter.getLastUpdateTime(), fileTriggerRouter.getFileTrigger().getTriggerId(), fileTriggerRouter.getRouter().getRouterId() }, new int[] { Types.SMALLINT, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR })) { fileTriggerRouter.setCreateTime(fileTriggerRouter.getLastUpdateTime()); sqlTemplate.update(getSql("insertFileTriggerRouterSql"), new Object[] { fileTriggerRouter.isEnabled() ? 1 : 0, fileTriggerRouter.isInitialLoadEnabled() ? 1 : 0, fileTriggerRouter.getTargetBaseDir(), fileTriggerRouter.getConflictStrategy().name(), fileTriggerRouter.getCreateTime(), fileTriggerRouter.getLastUpdateBy(), fileTriggerRouter.getLastUpdateTime(), fileTriggerRouter.getFileTrigger().getTriggerId(), fileTriggerRouter.getRouter().getRouterId() }, new int[] { Types.SMALLINT, Types.SMALLINT, Types.VARCHAR, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR }); } } public void deleteFileTriggerRouter(String triggerId, String routerId) { sqlTemplate.update(getSql("deleteFileTriggerRouterSql"), triggerId, routerId); } public void deleteFileTriggerRouter(FileTriggerRouter fileTriggerRouter) { sqlTemplate.update(getSql("deleteFileTriggerRouterSql"), (Object) fileTriggerRouter.getFileTrigger().getTriggerId(), fileTriggerRouter.getRouter().getRouterId()); } public void deleteFileTrigger(FileTrigger fileTrigger) { sqlTemplate.update(getSql("deleteFileTriggerSql"), (Object) fileTrigger.getTriggerId()); } public List<FileTriggerRouter> getFileTriggerRouters(FileTrigger fileTrigger) { return sqlTemplate.query(getSql("selectFileTriggerRoutersSql", "whereTriggerIdSql"), new FileTriggerRouterMapper(), fileTrigger.getTriggerId()); } public DirectorySnapshot getDirectorySnapshot(FileTriggerRouter fileTriggerRouter) { return new DirectorySnapshot(fileTriggerRouter, sqlTemplate.query(getSql("selectFileSnapshotSql"), new FileSnapshotMapper(), fileTriggerRouter.getFileTrigger().getTriggerId(), fileTriggerRouter.getRouter().getRouterId())); } public void save(List<FileSnapshot> changes) { if (changes != null) { ISqlTransaction sqlTransaction = null; try { sqlTransaction = sqlTemplate.startSqlTransaction(); for (FileSnapshot fileSnapshot : changes) { save(sqlTransaction, fileSnapshot); } sqlTransaction.commit(); } catch (Error ex) { if (sqlTransaction != null) { sqlTransaction.rollback(); } throw ex; } catch (RuntimeException ex) { if (sqlTransaction != null) { sqlTransaction.rollback(); } throw ex; } finally { close(sqlTransaction); } } } public void save(ISqlTransaction sqlTransaction, FileSnapshot snapshot) { snapshot.setLastUpdateTime(new Date()); if (0 == sqlTransaction.prepareAndExecute(getSql("updateFileSnapshotSql"), new Object[] { snapshot.getLastEventType().getCode(), snapshot.getCrc32Checksum(), snapshot.getFileSize(), snapshot.getFileModifiedTime(), snapshot.getLastUpdateTime(), snapshot.getLastUpdateBy(), snapshot.getChannelId(), snapshot.getReloadChannelId(), snapshot.getTriggerId(), snapshot.getRouterId(), snapshot.getRelativeDir(), snapshot.getFileName() }, new int[] { Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR })) { snapshot.setCreateTime(snapshot.getLastUpdateTime()); sqlTransaction.prepareAndExecute(getSql("insertFileSnapshotSql"), new Object[] { snapshot.getLastEventType().getCode(), snapshot.getCrc32Checksum(), snapshot.getFileSize(), snapshot.getFileModifiedTime(), snapshot.getCreateTime(), snapshot.getLastUpdateTime(), snapshot.getLastUpdateBy(), snapshot.getChannelId(), snapshot.getReloadChannelId(), snapshot.getTriggerId(), snapshot.getRouterId(), snapshot.getRelativeDir(), snapshot.getFileName() }, new int[] { Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP, Types.TIMESTAMP, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR }); } // now that we have captured an update, delete the row for cleanup if (snapshot.getLastEventType() == LastEventType.DELETE) { sqlTransaction.prepareAndExecute(getSql("deleteFileSnapshotSql"), new Object[] { snapshot.getTriggerId(), snapshot.getRouterId(), snapshot.getRelativeDir(), snapshot.getFileName() }, new int[] { Types.VARCHAR, Types.VARCHAR, Types.VARCHAR, Types.VARCHAR }); } } synchronized public RemoteNodeStatuses pullFilesFromNodes(boolean force) { return queueJob(force, parameterService.getLong(ParameterConstants.FILE_PULL_MINIMUM_PERIOD_MS, -1), ClusterConstants.FILE_SYNC_PULL, CommunicationType.FILE_PULL); } synchronized public RemoteNodeStatuses pushFilesToNodes(boolean force) { return queueJob(force, parameterService.getLong(ParameterConstants.FILE_PUSH_MINIMUM_PERIOD_MS, -1), ClusterConstants.FILE_SYNC_PUSH, CommunicationType.FILE_PUSH); } public List<OutgoingBatch> sendFiles(ProcessInfo processInfo, Node targetNode, IOutgoingTransport outgoingTransport) { List<OutgoingBatch> processedBatches = new ArrayList<OutgoingBatch>(); List<OutgoingBatch> batchesToProcess = new ArrayList<OutgoingBatch>(); List<Channel> fileSyncChannels = engine.getConfigurationService().getFileSyncChannels(); for (Channel channel : fileSyncChannels) { OutgoingBatches batches = engine.getOutgoingBatchService().getOutgoingBatches(targetNode.getNodeId(), false); batchesToProcess.addAll(batches.filterBatchesForChannel(channel)); } OutgoingBatch currentBatch = null; IStagingManager stagingManager = engine.getStagingManager(); long memoryThresholdInBytes = parameterService.getLong(ParameterConstants.STREAM_TO_FILE_THRESHOLD); IStagedResource stagedResource = stagingManager.create(memoryThresholdInBytes, Constants.STAGING_CATEGORY_OUTGOING, processInfo.getSourceNodeId(), targetNode.getNodeId(), "filesync.zip"); try { long maxBytesToSync = parameterService.getLong(ParameterConstants.TRANSPORT_MAX_BYTES_TO_SYNC); FileSyncZipDataWriter dataWriter = new FileSyncZipDataWriter(maxBytesToSync, this, engine.getNodeService(), stagedResource); try { for (int i = 0; i < batchesToProcess.size(); i++) { currentBatch = batchesToProcess.get(i); processInfo.incrementBatchCount(); processInfo.setCurrentBatchId(currentBatch.getBatchId()); ((DataExtractorService) engine.getDataExtractorService()).extractOutgoingBatch(processInfo, targetNode, dataWriter, currentBatch, false, true, DataExtractorService.ExtractMode.FOR_SYM_CLIENT); processedBatches.add(currentBatch); /* * check to see if max bytes to sync has been reached and * stop processing batches */ if (dataWriter.readyToSend()) { break; } } } finally { dataWriter.finish(); } processInfo.setStatus(ProcessInfo.Status.TRANSFERRING); for (int i = 0; i < batchesToProcess.size(); i++) { batchesToProcess.get(i).setStatus(Status.SE); } engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess); try { if (stagedResource.exists()) { InputStream is = stagedResource.getInputStream(); try { OutputStream os = outgoingTransport.openStream(); IOUtils.copy(is, os); os.flush(); } catch (IOException e) { throw new IoException(e); } } for (int i = 0; i < batchesToProcess.size(); i++) { batchesToProcess.get(i).setStatus(Status.LD); } engine.getOutgoingBatchService().updateOutgoingBatches(batchesToProcess); } finally { stagedResource.close(); } } catch (RuntimeException e) { if (currentBatch != null) { engine.getStatisticManager().incrementDataExtractedErrors(currentBatch.getChannelId(), 1); currentBatch.setSqlMessage(getRootMessage(e)); currentBatch.revertStatsOnError(); if (currentBatch.getStatus() != Status.IG) { currentBatch.setStatus(Status.ER); } currentBatch.setErrorFlag(true); engine.getOutgoingBatchService().updateOutgoingBatch(currentBatch); if (isStreamClosedByClient(e)) { log.warn("Failed to extract batch {}. The stream was closed by the client. The error was: {}", currentBatch, getRootMessage(e)); } else { log.error("Failed to extract batch {}", currentBatch, e); } } else { log.error("Could not log the outgoing batch status because the batch was null", e); } throw e; } finally { if (stagedResource != null) { stagedResource.delete(); } } return processedBatches; } public void acknowledgeFiles(OutgoingBatch outgoingBatch) { log.debug("Acknowledging file_sync outgoing batch-{}", outgoingBatch.getBatchId()); ISqlReadCursor<Data> cursor = engine.getDataService().selectDataFor(outgoingBatch.getBatchId(), outgoingBatch.getChannelId()); Data data = null; List<File> filesToDelete = new ArrayList<File>(); Table snapshotTable = platform.getTableFromCache( TableConstants.getTableName(tablePrefix, TableConstants.SYM_FILE_SNAPSHOT), false); for (int i = 0; i < outgoingBatch.getInsertEventCount(); i++) { data = cursor.next(); if (data != null && (data.getDataEventType() == DataEventType.INSERT || data.getDataEventType() == DataEventType.UPDATE)) { Map<String, String> columnData = data.toColumnNameValuePairs(snapshotTable.getColumnNames(), CsvData.ROW_DATA); FileSnapshot fileSnapshot = new FileSnapshot(); fileSnapshot.setTriggerId(columnData.get("TRIGGER_ID")); fileSnapshot.setRouterId(columnData.get("ROUTER_ID")); fileSnapshot.setFileModifiedTime(Long.parseLong(columnData.get("FILE_MODIFIED_TIME"))); fileSnapshot.setFileName(columnData.get("FILE_NAME")); fileSnapshot.setRelativeDir(columnData.get("RELATIVE_DIR")); fileSnapshot.setLastEventType(LastEventType.fromCode(columnData.get("LAST_EVENT_TYPE"))); FileTriggerRouter triggerRouter = this.getFileTriggerRouter(fileSnapshot.getTriggerId(), fileSnapshot.getRouterId()); if (triggerRouter != null) { FileTrigger fileTrigger = triggerRouter.getFileTrigger(); if (fileTrigger.isDeleteAfterSync()) { File file = fileTrigger.createSourceFile(fileSnapshot); if (!file.isDirectory()) { filesToDelete.add(file); if (fileTrigger.isSyncOnCtlFile()) { filesToDelete.add(new File(file.getAbsolutePath() + ".ctl")); } } } } } } if (cursor != null) { cursor.close(); cursor = null; } if (filesToDelete != null && filesToDelete.size() > 0) { for (File file : filesToDelete) { if (file != null && file.exists()) { log.debug("Deleting the '{}' file", file.getAbsolutePath()); boolean deleted = FileUtils.deleteQuietly(file); if (!deleted) { log.warn("Failed to 'delete on sync' the {} file", file.getAbsolutePath()); } } file = null; } filesToDelete = null; } } public void loadFilesFromPush(String nodeId, InputStream in, OutputStream out) { INodeService nodeService = engine.getNodeService(); Node local = nodeService.findIdentity(); Node sourceNode = nodeService.findNode(nodeId); if (local != null && sourceNode != null) { ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo( new ProcessInfoKey(nodeId, local.getNodeId(), ProcessType.FILE_SYNC_PUSH_HANDLER)); try { List<IncomingBatch> list = processZip(in, nodeId, processInfo); NodeSecurity security = nodeService.findNodeSecurity(local.getNodeId()); processInfo.setStatus(ProcessInfo.Status.ACKING); engine.getTransportManager().writeAcknowledgement(out, sourceNode, list, local, security != null ? security.getNodePassword() : null); processInfo.setStatus(ProcessInfo.Status.OK); } catch (Throwable e) { processInfo.setStatus(ProcessInfo.Status.ERROR); if (e instanceof IOException) { throw new IoException((IOException) e); } else if (e instanceof RuntimeException) { throw (RuntimeException) e; } else { throw new RuntimeException(e); } } } else { throw new SymmetricException("Could not load data because the node is not registered"); } } public void execute(NodeCommunication nodeCommunication, RemoteNodeStatus status) { Node identity = engine.getNodeService().findIdentity(); if (identity != null) { NodeSecurity security = engine.getNodeService().findNodeSecurity(identity.getNodeId()); if (security != null) { if (nodeCommunication.getCommunicationType() == CommunicationType.FILE_PULL) { pullFilesFromNode(nodeCommunication, status, identity, security); } else if (nodeCommunication.getCommunicationType() == CommunicationType.FILE_PUSH) { pushFilesToNode(nodeCommunication, status, identity, security); } } } } protected void pushFilesToNode(NodeCommunication nodeCommunication, RemoteNodeStatus status, Node identity, NodeSecurity security) { ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo(new ProcessInfoKey( nodeCommunication.getNodeId(), identity.getNodeId(), ProcessType.FILE_SYNC_PUSH_JOB)); IOutgoingWithResponseTransport transport = null; try { transport = engine.getTransportManager().getFilePushTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), parameterService.getRegistrationUrl()); List<OutgoingBatch> batches = sendFiles(processInfo, nodeCommunication.getNode(), transport); if (batches.size() > 0) { List<BatchAck> batchAcks = readAcks(batches, transport, engine.getTransportManager(), engine.getAcknowledgeService()); status.updateOutgoingStatus(batches, batchAcks); } } catch (Exception e) { fireOffline(e, nodeCommunication.getNode(), status); } finally { if (transport != null) { transport.close(); } if (processInfo.getStatus() != ProcessInfo.Status.ERROR) { processInfo.setStatus(ProcessInfo.Status.OK); } } } protected List<IncomingBatch> processZip(InputStream is, String sourceNodeId, ProcessInfo processInfo) throws IOException { File unzipDir = new File(parameterService.getTempDirectory(), String.format("filesync_incoming/%s/%s", engine.getNodeService().findIdentityNodeId(), sourceNodeId)); FileUtils.deleteDirectory(unzipDir); unzipDir.mkdirs(); AppUtils.unzip(is, unzipDir); Set<Long> batchIds = new TreeSet<Long>(); String[] files = unzipDir.list(DirectoryFileFilter.INSTANCE); if (files != null) { for (int i = 0; i < files.length; i++) { try { batchIds.add(Long.parseLong(files[i])); } catch (NumberFormatException e) { log.error( "Unexpected directory name. Expected a number representing a batch id. Instead the directory was named '{}'", files[i]); } } } List<IncomingBatch> batchesProcessed = new ArrayList<IncomingBatch>(); IIncomingBatchService incomingBatchService = engine.getIncomingBatchService(); processInfo.setStatus(ProcessInfo.Status.LOADING); for (Long batchId : batchIds) { processInfo.setCurrentBatchId(batchId); processInfo.incrementBatchCount(); File batchDir = new File(unzipDir, Long.toString(batchId)); IncomingBatch incomingBatch = new IncomingBatch(); File batchInfo = new File(batchDir, "batch-info.txt"); if (batchInfo.exists()) { List<String> info = FileUtils.readLines(batchInfo); if (info != null && info.size() > 0) { incomingBatch.setChannelId(info.get(0).trim()); } else { incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC); } } else { incomingBatch.setChannelId(Constants.CHANNEL_FILESYNC); } incomingBatch.setBatchId(batchId); incomingBatch.setStatus(IncomingBatch.Status.LD); incomingBatch.setNodeId(sourceNodeId); incomingBatch.setByteCount(FileUtils.sizeOfDirectory(batchDir)); batchesProcessed.add(incomingBatch); if (incomingBatchService.acquireIncomingBatch(incomingBatch)) { File syncScript = new File(batchDir, "sync.bsh"); if (syncScript.exists()) { String script = FileUtils.readFileToString(syncScript); Interpreter interpreter = new Interpreter(); boolean isLocked = false; try { interpreter.set("log", log); interpreter.set("batchDir", batchDir.getAbsolutePath().replace('\\', '/')); interpreter.set("engine", engine); interpreter.set("sourceNodeId", sourceNodeId); long waitMillis = getParameterService().getLong(ParameterConstants.FILE_SYNC_LOCK_WAIT_MS); log.debug("The {} node is attempting to get shared lock for to update incoming status", sourceNodeId); isLocked = engine.getClusterService().lock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED, waitMillis); if (isLocked) { log.debug("The {} node got a shared file sync lock", sourceNodeId); @SuppressWarnings("unchecked") Map<String, String> filesToEventType = (Map<String, String>) interpreter.eval(script); updateFileIncoming(sourceNodeId, filesToEventType); incomingBatch.setStatementCount(filesToEventType != null ? filesToEventType.size() : 0); } else { throw new RuntimeException( "Could not obtain file sync shared lock within " + waitMillis + " millis"); } incomingBatch.setStatus(IncomingBatch.Status.OK); if (incomingBatchService.isRecordOkBatchesEnabled()) { incomingBatchService.updateIncomingBatch(incomingBatch); } else if (incomingBatch.isRetry()) { incomingBatchService.deleteIncomingBatch(incomingBatch); } } catch (Throwable ex) { if (ex instanceof TargetError) { Throwable target = ((TargetError) ex).getTarget(); if (target != null) { ex = target; } } else if (ex instanceof EvalError) { log.error("Failed to evalulate the script:\n{}", script); } if (ex instanceof FileConflictException) { log.error(ex.getMessage() + ". Failed to process file sync batch " + batchId); } else { log.error("Failed to process file sync batch " + batchId, ex); } incomingBatch.setErrorFlag(true); incomingBatch.setStatus(IncomingBatch.Status.ER); incomingBatch.setSqlMessage(ex.getMessage()); if (incomingBatchService.isRecordOkBatchesEnabled() || incomingBatch.isRetry()) { incomingBatchService.updateIncomingBatch(incomingBatch); } else { incomingBatchService.insertIncomingBatch(incomingBatch); } processInfo.setStatus(ProcessInfo.Status.ERROR); break; } finally { log.debug("The {} node is done processing file sync files", sourceNodeId); if (isLocked) { engine.getClusterService().unlock(ClusterConstants.FILE_SYNC_SHARED, ClusterConstants.TYPE_SHARED); } } } else { log.error("Could not find the sync.bsh script for batch {}", batchId); } } } return batchesProcessed; } protected void updateFileIncoming(String nodeId, Map<String, String> filesToEventType) { Set<String> filePaths = filesToEventType.keySet(); for (String filePath : filePaths) { String eventType = filesToEventType.get(filePath); File file = new File(filePath); String fileName = file.getName(); String dirName = file.getParentFile().getPath().replace('\\', '/'); long lastUpdateTime = file.lastModified(); int updateCount = sqlTemplate.update(getSql("updateFileIncoming"), nodeId, lastUpdateTime, eventType, dirName, fileName); if (updateCount == 0) { sqlTemplate.update(getSql("insertFileIncoming"), nodeId, lastUpdateTime, eventType, dirName, fileName); } } } protected void pullFilesFromNode(NodeCommunication nodeCommunication, RemoteNodeStatus status, Node identity, NodeSecurity security) { IIncomingTransport transport = null; ProcessInfo processInfo = engine.getStatisticManager().newProcessInfo(new ProcessInfoKey( nodeCommunication.getNodeId(), identity.getNodeId(), ProcessType.FILE_SYNC_PULL_JOB)); try { processInfo.setStatus(ProcessInfo.Status.TRANSFERRING); transport = engine.getTransportManager().getFilePullTransport(nodeCommunication.getNode(), identity, security.getNodePassword(), null, parameterService.getRegistrationUrl()); List<IncomingBatch> batchesProcessed = processZip(transport.openStream(), nodeCommunication.getNodeId(), processInfo); if (batchesProcessed.size() > 0) { processInfo.setStatus(ProcessInfo.Status.ACKING); status.updateIncomingStatus(batchesProcessed); sendAck(nodeCommunication.getNode(), identity, security, batchesProcessed, engine.getTransportManager()); } } catch (Exception e) { fireOffline(e, nodeCommunication.getNode(), status); } finally { if (transport != null) { transport.close(); } if (processInfo.getStatus() != ProcessInfo.Status.ERROR) { processInfo.setStatus(ProcessInfo.Status.OK); } } } protected RemoteNodeStatuses queueJob(boolean force, long minimumPeriodMs, String clusterLock, CommunicationType type) { final RemoteNodeStatuses statuses = new RemoteNodeStatuses( engine.getConfigurationService().getChannels(false)); Node identity = engine.getNodeService().findIdentity(false); if (identity != null && identity.isSyncEnabled()) { if (force || !engine.getClusterService().isInfiniteLocked(clusterLock)) { INodeCommunicationService nodeCommunicationService = engine.getNodeCommunicationService(); List<NodeCommunication> nodes = nodeCommunicationService.list(type); int availableThreads = nodeCommunicationService.getAvailableThreads(type); for (NodeCommunication nodeCommunication : nodes) { if (StringUtils.isNotBlank(nodeCommunication.getNode().getSyncUrl()) || !parameterService.isRegistrationServer()) { boolean meetsMinimumTime = true; if (minimumPeriodMs > 0 && nodeCommunication.getLastLockTime() != null && (System.currentTimeMillis() - nodeCommunication.getLastLockTime().getTime()) < minimumPeriodMs) { meetsMinimumTime = false; } if (availableThreads > 0 && meetsMinimumTime) { if (nodeCommunicationService.execute(nodeCommunication, statuses, this)) { availableThreads--; } } } else { log.warn( "File sync cannot communicate with node '{}' in the group '{}'. The sync url is blank", nodeCommunication.getNode().getNodeId(), nodeCommunication.getNode().getNodeGroupId()); } } } else { log.debug("Did not run the {} process because it has been stopped", type.name().toLowerCase()); } } return statuses; } class FileTriggerMapper implements ISqlRowMapper<FileTrigger> { public FileTrigger mapRow(Row rs) { FileTrigger fileTrigger = new FileTrigger(); fileTrigger.setBaseDir( rs.getString("base_dir") == null ? null : rs.getString("base_dir").replace('\\', '/')); fileTrigger.setCreateTime(rs.getDateTime("create_time")); fileTrigger.setExcludesFiles(rs.getString("excludes_files")); fileTrigger.setIncludesFiles(rs.getString("includes_files")); fileTrigger.setLastUpdateBy(rs.getString("last_update_by")); fileTrigger.setLastUpdateTime(rs.getDateTime("last_update_time")); fileTrigger.setRecurse(rs.getBoolean("recurse")); fileTrigger.setSyncOnCreate(rs.getBoolean("sync_on_create")); fileTrigger.setSyncOnDelete(rs.getBoolean("sync_on_delete")); fileTrigger.setAfterCopyScript(rs.getString("after_copy_script")); fileTrigger.setBeforeCopyScript(rs.getString("before_copy_script")); fileTrigger.setSyncOnModified(rs.getBoolean("sync_on_modified")); fileTrigger.setSyncOnCtlFile(rs.getBoolean("sync_on_ctl_file")); fileTrigger.setDeleteAfterSync(rs.getBoolean("delete_after_sync")); fileTrigger.setTriggerId(rs.getString("trigger_id")); fileTrigger.setChannelId(rs.getString("channel_id")); fileTrigger.setReloadChannelId(rs.getString("reload_channel_id")); return fileTrigger; } } class FileTriggerRouterMapper implements ISqlRowMapper<FileTriggerRouter> { public FileTriggerRouter mapRow(Row rs) { FileTriggerRouter fileTriggerRouter = new FileTriggerRouter(); String triggerId = rs.getString("trigger_id"); FileTrigger fileTrigger = getFileTrigger(triggerId); fileTriggerRouter.setFileTrigger(fileTrigger); fileTriggerRouter.setConflictStrategy( FileConflictStrategy.valueOf(rs.getString("conflict_strategy").toUpperCase())); fileTriggerRouter.setCreateTime(rs.getDateTime("create_time")); fileTriggerRouter.setLastUpdateBy(rs.getString("last_update_by")); fileTriggerRouter.setLastUpdateTime(rs.getDateTime("last_update_time")); fileTriggerRouter.setEnabled(rs.getBoolean("enabled")); fileTriggerRouter.setInitialLoadEnabled(rs.getBoolean("initial_load_enabled")); fileTriggerRouter.setTargetBaseDir((rs.getString("target_base_dir") == null) ? null : rs.getString("target_base_dir").replace('\\', '/')); fileTriggerRouter .setRouter(engine.getTriggerRouterService().getRouterById(true, rs.getString("router_id"))); return fileTriggerRouter; } } class FileSnapshotMapper implements ISqlRowMapper<FileSnapshot> { public FileSnapshot mapRow(Row rs) { FileSnapshot fileSnapshot = new FileSnapshot(); fileSnapshot.setCrc32Checksum(rs.getLong("crc32_checksum")); fileSnapshot.setCreateTime(rs.getDateTime("create_time")); fileSnapshot.setChannelId(rs.getString("channel_id")); fileSnapshot.setReloadChannelId(rs.getString("reload_channel_id")); fileSnapshot.setLastUpdateBy(rs.getString("last_update_by")); fileSnapshot.setLastUpdateTime(rs.getDateTime("last_update_time")); fileSnapshot.setFileModifiedTime(rs.getLong("file_modified_time")); fileSnapshot.setFileName(rs.getString("file_name")); fileSnapshot.setRelativeDir( rs.getString("relative_dir") == null ? null : rs.getString("relative_dir").replace('\\', '/')); fileSnapshot.setFileSize(rs.getLong("file_size")); fileSnapshot.setLastEventType(LastEventType.fromCode(rs.getString("last_event_type"))); fileSnapshot.setTriggerId(rs.getString("trigger_id")); fileSnapshot.setRouterId(rs.getString("router_id")); return fileSnapshot; } } }