Java tutorial
/* * Copyright (C) 2010-2101 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.otter.node.etl.conflict.impl; import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; import org.springframework.util.CollectionUtils; import com.alibaba.otter.node.common.communication.NodeCommmunicationClient; import com.alibaba.otter.node.common.config.ConfigClientService; import com.alibaba.otter.node.etl.OtterConstants; import com.alibaba.otter.node.etl.conflict.FileBatchConflictDetectService; import com.alibaba.otter.node.etl.conflict.model.ConflictEventType; import com.alibaba.otter.node.etl.conflict.model.FileConflictDetectEvent; import com.alibaba.otter.node.etl.load.loader.db.FileloadDumper; import com.alibaba.otter.shared.common.model.config.pipeline.Pipeline; import com.alibaba.otter.shared.common.utils.thread.ExecutorTemplate; import com.alibaba.otter.shared.common.utils.thread.ExecutorTemplateGetter; import com.alibaba.otter.shared.communication.core.CommunicationRegistry; import com.alibaba.otter.shared.etl.model.EventType; import com.alibaba.otter.shared.etl.model.FileBatch; import com.alibaba.otter.shared.etl.model.FileData; /** * ???? * * @author jianghang 2011-11-10 ?09:41:20 * @version 4.0.0 */ public class FileBatchConflictDetectServiceImpl implements FileBatchConflictDetectService { private static final Logger logger = LoggerFactory.getLogger(FileBatchConflictDetectServiceImpl.class); private int retry = 3; private ConfigClientService configClientService; private NodeCommmunicationClient nodeCommmunicationClient; private ExecutorTemplateGetter executorTemplateGetter; public FileBatchConflictDetectServiceImpl() { // ? CommunicationRegistry.regist(ConflictEventType.fileConflictDetect, this); } public FileBatch detect(FileBatch fileBatch, Long targetNodeId) { FileConflictDetectEvent event = new FileConflictDetectEvent(); event.setFileBatch(fileBatch); if (isLocal(targetNodeId)) { return onFileConflictDetect(event); } else { // return (FileBatch) nodeCommmunicationClient.call(targetNodeId, event); } } /** * ? */ private FileBatch onFileConflictDetect(FileConflictDetectEvent event) { final FileBatch fileBatch = event.getFileBatch(); if (CollectionUtils.isEmpty(fileBatch.getFiles())) { return fileBatch; } ExecutorTemplate executorTemplate = executorTemplateGetter.get(); try { MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId())); executorTemplate.start(); // ?poolSize Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId()); executorTemplate.adjustPoolSize(pipeline.getParameters().getFileLoadPoolSize()); // ? final List<FileData> result = Collections.synchronizedList(new ArrayList<FileData>()); final List<FileData> filter = Collections.synchronizedList(new ArrayList<FileData>()); for (final FileData source : fileBatch.getFiles()) { EventType type = source.getEventType(); if (type.isDelete()) { result.add(source); } else { executorTemplate.submit(new Runnable() { public void run() { MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId())); // ? String namespace = source.getNameSpace(); String path = source.getPath(); FileData target = null; int count = 0; while (count++ < retry) {// ?? try { if (true == StringUtils.isBlank(namespace)) { // local file java.io.File targetFile = new java.io.File(path); if (true == targetFile.exists()) { // modified time cost long lastModified = targetFile.lastModified(); long size = targetFile.length(); // ? target = new FileData(); target.setLastModifiedTime(lastModified); target.setSize(size); } } else { // remote file throw new RuntimeException(source + " is not support!"); } break; // ? } catch (Exception ex) { target = null; } } boolean shouldSync = false; if (target != null) { if (true == accept(target, source)) { shouldSync = true; } } else { shouldSync = true; } if (true == shouldSync) { result.add(source); } else { filter.add(source); } } }); } } // ?? executorTemplate.waitForResult(); if (pipeline.getParameters().getDumpEvent() && logger.isInfoEnabled()) { logger.info(FileloadDumper.dumpFilterFileDatas(fileBatch.getIdentity(), fileBatch.getFiles().size(), result.size(), filter)); } // FileBatch target = new FileBatch(); target.setIdentity(fileBatch.getIdentity()); target.setFiles(result); return target; } finally { if (executorTemplate != null) { executorTemplateGetter.release(executorTemplate); } MDC.remove(OtterConstants.splitPipelineLoadLogFileKey); } } /** * <pre> * * 1. ??? * 2. ??? * </pre> */ private boolean accept(FileData target, FileData source) { return (target.getLastModifiedTime() < source.getLastModifiedTime()) || (target.getSize() != source.getSize()); } private boolean isLocal(Long targetNodeId) { return configClientService.currentNode().getId().equals(targetNodeId); } // ===================== setter / getter ====================== public void setConfigClientService(ConfigClientService configClientService) { this.configClientService = configClientService; } public void setNodeCommmunicationClient(NodeCommmunicationClient nodeCommmunicationClient) { this.nodeCommmunicationClient = nodeCommmunicationClient; } public void setRetry(int retry) { this.retry = retry; } public void setExecutorTemplateGetter(ExecutorTemplateGetter executorTemplateGetter) { this.executorTemplateGetter = executorTemplateGetter; } }