Java tutorial
/** * Project: doris.config.server-1.0-SNAPSHOT * * File Created at 2011-4-27 * $Id$ * * Copyright 1999-2100 Alibaba.com Corporation Limited. * All rights reserved. * * This software is the confidential and proprietary information of * Alibaba Company. ("Confidential Information"). You shall not * disclose such Confidential Information and shall use it only in * accordance with the terms of the license agreement you entered into * with Alibaba.com. */ package com.alibaba.doris.admin.service.common.route; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.alibaba.doris.admin.core.AdminServiceLocator; import com.alibaba.doris.admin.dataobject.PhysicalNodeDO; import com.alibaba.doris.admin.dataobject.RouterConfigInstanceDO; import com.alibaba.doris.admin.monitor.MonitorEnum; import com.alibaba.doris.admin.monitor.SystemLogMonitor; import com.alibaba.doris.admin.service.AdminNodeService; import com.alibaba.doris.admin.service.PropertiesService; import com.alibaba.doris.admin.service.RouteConfigService; import com.alibaba.doris.admin.service.common.Managerable; import com.alibaba.doris.common.AdminServiceConstants; import com.alibaba.doris.common.MonitorWarnConstants; import com.alibaba.doris.common.NodeRouteStatus; import com.alibaba.doris.common.StoreNode; import com.alibaba.doris.common.StoreNodeSequenceEnum; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; /** * @author mianhe */ public class RouteConfigProcessor implements Managerable { private static final Log logger = LogFactory.getLog(RouteConfigProcessor.class); private volatile boolean initialized = false; private Thread configLoadThread; private volatile boolean stopped = false; private AdminNodeService nodeService = AdminServiceLocator.getAdminNodeService(); private volatile RouterConfigInstanceDO currentConfigInstanceDo = null; private volatile Map<Integer, PhysicalNodeDO> currentPhysicalNodeMap = new ConcurrentHashMap<Integer, PhysicalNodeDO>(); private static final RouteConfigProcessor instance = new RouteConfigProcessor(); private RouteConfigService routeConfigService = AdminServiceLocator.getRouteConfigService(); private static PropertiesService propertyService = AdminServiceLocator.getPropertiesService(); private static final long routeConfigScanInterval = propertyService.getProperty("routeConfigScanInterval", Long.TYPE, AdminServiceConstants.ROUTER_SCAN_DEFAULT_RELOAD_INTERVAL); private RouteConfigProcessor() { try { init(); } catch (DorisConfigServiceException e) { SystemLogMonitor.error(MonitorEnum.ROUTER, MonitorWarnConstants.RE_GEN_ROUTE_FAILED, e); logger.error("failed to init " + RouteConfigProcessor.class, e); } } public static RouteConfigProcessor getInstance() { return instance; } private synchronized void init() throws DorisConfigServiceException { if (initialized == true) { if (logger.isInfoEnabled()) { logger.warn("RouteConfigCachableProcessor is already initialized."); } return; } //?load??? load(); //?? configLoadThread = new Thread(new LoadTask(), "load-config-instance-thread"); configLoadThread.setDaemon(true); //configLoadThread.start(); this.initialized = true; } /** * @throws DorisConfigServiceException */ private void load() throws DorisConfigServiceException { //? Collection<PhysicalNodeDO> latestPhysicalNodeList = loadValidNodes(); // fill map boolean filled = fillPhysicalNodeMap(latestPhysicalNodeList, false); if (filled) { currentConfigInstanceDo = routeConfigService.loadLatestConfigInstance(); } if (currentConfigInstanceDo == null) { insertNewConfigInstance(false); } } private Collection<PhysicalNodeDO> loadValidNodes() throws DorisConfigServiceException { Collection<PhysicalNodeDO> latestPhysicalNodeList = nodeService.queryAllPhysicalNodes(); //filter this.filterNodes(latestPhysicalNodeList); //valid this.validateNodes(latestPhysicalNodeList); return latestPhysicalNodeList; } private boolean fillPhysicalNodeMap(Collection<PhysicalNodeDO> latestPhysicalNodeList, boolean clearmap) { if (clearmap) { currentPhysicalNodeMap.clear(); } if (currentPhysicalNodeMap.isEmpty()) { if (latestPhysicalNodeList != null && !latestPhysicalNodeList.isEmpty()) { for (PhysicalNodeDO latestNode : latestPhysicalNodeList) { if (isValidRoutableNode(latestNode)) { currentPhysicalNodeMap.put(latestNode.getId(), latestNode); } } } else { //??null?empty,??? logger.warn("Cannot load doris config, DORIS config database may not have config data."); } return true; } return false; } private void insertNewConfigInstance(boolean needCheckDB) { if (needCheckDB) { RouterConfigInstanceDO newConfigInstance = buildNewConfigInstance(currentPhysicalNodeMap); // ???Config Instance ?? RouterConfigInstanceDO latestConfigFromDb = routeConfigService.loadLatestConfigInstance(); List<StoreNode> newStoreNodes = JSON.parseArray(newConfigInstance.getContent(), StoreNode.class); List<StoreNode> dbStoreNodes = JSON.parseArray(latestConfigFromDb.getContent(), StoreNode.class); long begin = System.currentTimeMillis(); boolean identical = checkIdentical(newStoreNodes, dbStoreNodes); if (logger.isWarnEnabled()) { logger.warn("indential compared cost:" + (System.currentTimeMillis() - begin)); } if (identical) { currentConfigInstanceDo = latestConfigFromDb; if (logger.isWarnEnabled()) { logger.warn("new config has already generated!!!"); } } else { int id = routeConfigService.insertConfigInstance(newConfigInstance); newConfigInstance.setId(id); currentConfigInstanceDo = routeConfigService.loadLatestConfigInstance(); if (logger.isWarnEnabled()) { int loadedid = currentConfigInstanceDo.getId(); if (id != loadedid) { logger.warn("Confilict route version occurs, use loaded id from db.[insert id:" + id + ", loaded id:" + loadedid + "]"); } logger.warn("current config id:" + id); } } } else { RouterConfigInstanceDO newConfigInstance = buildNewConfigInstance(currentPhysicalNodeMap); int id = routeConfigService.insertConfigInstance(newConfigInstance); newConfigInstance.setId(id); currentConfigInstanceDo = routeConfigService.loadLatestConfigInstance(); if (logger.isWarnEnabled()) { int loadedid = currentConfigInstanceDo.getId(); if (id != loadedid) { logger.warn("Confilict route version occurs, use loaded id from db.[insert id:" + id + ", loaded id:" + loadedid + "]"); } logger.warn("system init, new config id:" + loadedid); } } } private boolean checkIdentical(List<StoreNode> newStoreNodes, List<StoreNode> dbStoreNodes) { if ((newStoreNodes == null && dbStoreNodes != null) || (newStoreNodes != null && dbStoreNodes == null)) { return false; } if (newStoreNodes != null && dbStoreNodes != null) { if (newStoreNodes.size() != dbStoreNodes.size()) { return false; } Comparator<StoreNode> snComparator = new Comparator<StoreNode>() { public int compare(StoreNode o1, StoreNode o2) { return o1.getPhId().compareTo(o2.getPhId()); } }; Collections.sort(dbStoreNodes, snComparator); Collections.sort(newStoreNodes, snComparator); for (int i = 0; i < dbStoreNodes.size(); i++) { StoreNode dbNode = dbStoreNodes.get(i); StoreNode newNode = newStoreNodes.get(i); boolean isDifferent = isDifferent(dbNode, newNode); if (isDifferent) { return false; } } } return true; } public void close() { if (!initialized) { if (logger.isInfoEnabled()) { logger.warn("The RouteConfigCachableProcessor is not initialized yet."); } return; } if (logger.isInfoEnabled()) { logger.info("RouteConfigCachableProcessor is closing!!!"); } stopped = true; if (logger.isInfoEnabled()) { logger.info("Thread of config refresh is stopping!!!"); } if (configLoadThread != null) { configLoadThread.interrupt(); } } /** * @return the currentConfigInstanceDo * @throws DorisConfigServiceException */ public RouterConfigInstanceDO getCurrentConfigInstanceDo() throws DorisConfigServiceException { if (!initialized) { if (logger.isInfoEnabled()) { logger.warn("The RouteConfigCachableProcessor is not initialized yet."); } } if (currentConfigInstanceDo == null) { //refresh if (logger.isDebugEnabled()) { logger.debug("cannot find configinstance:refresh."); } refresh(); } return currentConfigInstanceDo; } public synchronized void refreshWithDbLatest() throws DorisConfigServiceException { if (logger.isWarnEnabled()) { logger.warn("forced refresh starts..."); } RouterConfigInstanceDO dbConfigInstanceDo = routeConfigService.loadLatestConfigInstance(); if (currentConfigInstanceDo == null || (dbConfigInstanceDo.getId() > currentConfigInstanceDo.getId())) { currentConfigInstanceDo = dbConfigInstanceDo; Collection<PhysicalNodeDO> latestPhysicalNodeList = loadValidNodes(); fillPhysicalNodeMap(latestPhysicalNodeList, true); if (logger.isWarnEnabled()) { logger.warn("forced refreshed..., db config id :" + dbConfigInstanceDo.getId()); } } } public synchronized void refresh() throws DorisConfigServiceException { if (!initialized) { if (logger.isWarnEnabled()) { logger.warn("The RouteConfigCachableProcessor is not initialized yet."); } return; } Collection<PhysicalNodeDO> latestPhysicalNodeList = loadValidNodes(); boolean hasNodeChanged = checkNodeChanges(latestPhysicalNodeList); if (hasNodeChanged) { insertNewConfigInstance(true); } else if (currentConfigInstanceDo == null) { //??? if (logger.isWarnEnabled()) { logger.warn("The RouteConfigCachableProcessor is not initialized, reload."); } currentConfigInstanceDo = routeConfigService.loadLatestConfigInstance(); } else { // ?currentConfigInstanceDo } if (currentConfigInstanceDo == null) { logger.warn("Cannot load doris config, DORIS config database may not have config data."); } } private boolean checkNodeChanges(Collection<PhysicalNodeDO> latestPhysicalNodeList) { int latestPhysicalNodesNum = (latestPhysicalNodeList == null) ? 0 : latestPhysicalNodeList.size(); // ?? if (latestPhysicalNodesNum == 0) { if (currentPhysicalNodeMap.isEmpty()) { return false; } else { //currentPhysicalNodeMap currentPhysicalNodeMap.clear(); return true; } } // ??, nodecurrentPhysicalNodeMap boolean hasNodeChanged = false; if (latestPhysicalNodeList != null && !latestPhysicalNodeList.isEmpty()) { List<Integer> allLatestPhysicalNodeIds = new ArrayList<Integer>(); for (PhysicalNodeDO latestNode : latestPhysicalNodeList) { allLatestPhysicalNodeIds.add(latestNode.getId()); PhysicalNodeDO currentNode = currentPhysicalNodeMap.get(latestNode.getId()); if (currentNode == null || isChanged(latestNode, currentNode)) { currentPhysicalNodeMap.put(latestNode.getId(), latestNode); hasNodeChanged = true; } } for (Integer nodeId : currentPhysicalNodeMap.keySet()) { if (!allLatestPhysicalNodeIds.contains(nodeId)) { currentPhysicalNodeMap.remove(nodeId); hasNodeChanged = true; } } } return hasNodeChanged; } private void validateNodes(Collection<PhysicalNodeDO> allPhyNodes) throws DorisConfigServiceException { //for validate logical id Map<StoreNodeSequenceEnum, List<PhysicalNodeDO>> nodesGroup = new HashMap<StoreNodeSequenceEnum, List<PhysicalNodeDO>>(); // for validate nodes. Map<String, Set<StoreNodeSequenceEnum>> ipGroups = new HashMap<String, Set<StoreNodeSequenceEnum>>(); for (PhysicalNodeDO node : allPhyNodes) { StoreNodeSequenceEnum seq = StoreNodeSequenceEnum.getTypeByValue(node.getSerialId()); if (seq == null) { throw new InvalidNodeException("invalid sequence number for node with id :" + node.getId()); } if (StringUtils.isBlank(node.getIp())) { throw new InvalidNodeException("ip for node with id :" + node.getId()); } if (StringUtils.isBlank(node.getPhysicalId())) { throw new InvalidNodeException("invalid sequence number for node with id :" + node.getId()); } if (node.getPort() <= 0 || node.getPort() > 65535) { throw new InvalidNodeException("port is not valid for node with id :" + node.getId()); } if (StringUtils.isBlank(node.getMachineId())) { throw new InvalidNodeException("machine id is empty for node with id :" + node.getId()); } List<PhysicalNodeDO> nodes = nodesGroup.get(seq); if (nodes == null) { nodes = new ArrayList<PhysicalNodeDO>(); nodesGroup.put(seq, nodes); } nodes.add(node); Set<StoreNodeSequenceEnum> ips = ipGroups.get(node.getIp()); if (ips == null) { ips = new HashSet<StoreNodeSequenceEnum>(); } if (StoreNodeSequenceEnum.isNormalSequence(seq)) { ips.add(seq); } ipGroups.put(node.getIp(), ips); } // validate logic id in one sequence: for (Map.Entry<StoreNodeSequenceEnum, List<PhysicalNodeDO>> oneSequenceNodes : nodesGroup.entrySet()) { //valid the logical id in one sequence. List<PhysicalNodeDO> nodes = oneSequenceNodes.getValue(); List<Integer> allLogicalIds = new ArrayList<Integer>(); for (PhysicalNodeDO node : nodes) { allLogicalIds.add(node.getLogicalId()); } Collections.sort(allLogicalIds); for (int i = 0; i < allLogicalIds.size(); i++) { Integer logicalId = allLogicalIds.get(i); if (!logicalId.equals(i)) { throw new InvalidNodeException( "invalid logical id: " + logicalId + " in sequece :" + oneSequenceNodes.getKey()); } } } // the nodes in one machine cannot allocated to different normal sequence. for (Map.Entry<String, Set<StoreNodeSequenceEnum>> entry : ipGroups.entrySet()) { if (entry.getValue().size() > 1) { throw new InvalidNodeException("the nodes in machine (ip=" + entry.getKey() + ") is allocated in different sequence" + entry.getValue()); } } //Warning if there is no temp sequence. List<PhysicalNodeDO> tempNodes = nodesGroup.get(StoreNodeSequenceEnum.TEMP_SEQUENCE); if (tempNodes == null || tempNodes.isEmpty()) { SystemLogMonitor.error(MonitorEnum.ROUTER, MonitorWarnConstants.ROUTE_NO_TEMP_NODES); } } private void filterNodes(Collection<PhysicalNodeDO> latestPhysicalNodeList) { Iterator<PhysicalNodeDO> entryIter = latestPhysicalNodeList.iterator(); while (entryIter.hasNext()) { PhysicalNodeDO pNode = entryIter.next(); if (!isValidRoutableNode(pNode)) { entryIter.remove(); } } } private class LoadTask implements Runnable { public void run() { try { while (!stopped) { try { Thread.sleep(routeConfigScanInterval); } catch (Exception e) { logger.warn("Exception when the key fetch task sleep: " + e.toString()); } if (stopped) { return; } try { if (logger.isDebugEnabled()) { logger.debug("refresh route schedully."); } refresh(); } catch (Exception e) { logger.error("Exception when fetch doris config: ", e); SystemLogMonitor.error(MonitorEnum.ROUTER, MonitorWarnConstants.RE_GEN_ROUTE_FAILED, e); } } } finally { logger.warn("the key fetch thread exit!"); } } } private RouterConfigInstanceDO buildNewConfigInstance(Map<Integer, PhysicalNodeDO> currentPhysicalNodeMap) { RouterConfigInstanceDO configInstanceDO = new RouterConfigInstanceDO(); configInstanceDO.setGmtCreate(new Date()); String content = buildConfigInstanceContent(currentPhysicalNodeMap); configInstanceDO.setContent(content); return configInstanceDO; } private String buildConfigInstanceContent(Map<Integer, PhysicalNodeDO> currentPhysicalNodeMap) { List<StoreNode> storeNodeList = new ArrayList<StoreNode>(); if (currentPhysicalNodeMap == null || currentPhysicalNodeMap.isEmpty()) { logger.warn("Build config instance content, but the content might not be correct."); } else { for (PhysicalNodeDO node : currentPhysicalNodeMap.values()) { StoreNode storeNode = new StoreNode(); storeNode.setURL(node.getIp() + ":" + node.getPort()); storeNode.setIp(node.getIp()); storeNode.setPort(node.getPort()); storeNode.setLogicId(node.getLogicalId()); storeNode.setPhId(node.getPhysicalId()); storeNode.setSequence(StoreNodeSequenceEnum.getTypeByValue(node.getSerialId())); storeNode.setStatus(NodeRouteStatus.getTypeByValue(node.getStatus())); storeNodeList.add(storeNode); } } return JSON.toJSONString(storeNodeList, SerializerFeature.WriteEnumUsingToString); } private boolean isValidRoutableNode(PhysicalNodeDO latestNode) { if (latestNode == null) return false; NodeRouteStatus status = NodeRouteStatus.getTypeByValue(latestNode.getStatus()); StoreNodeSequenceEnum sequence = StoreNodeSequenceEnum.getTypeByValue(latestNode.getSerialId()); return isValidRoutableSequence(sequence) && isValidRoutableStatus(status); } private boolean isValidRoutableStatus(NodeRouteStatus status) { if (status == null) { return false; } return status == NodeRouteStatus.OK || status == NodeRouteStatus.TEMP_FAILED; } private boolean isValidRoutableSequence(StoreNodeSequenceEnum sequence) { if (sequence == null) return false; return (sequence != StoreNodeSequenceEnum.STANDBY_SEQUENCE) && (sequence != StoreNodeSequenceEnum.UNUSE_SEQUENCE); } private boolean isChanged(PhysicalNodeDO latestNode, PhysicalNodeDO currentNode) { // ?? return !latestNode.getMachineId().equals(currentNode.getMachineId()) //|| !latestNode.getGmtCreate().equals(currentNode.getGmtCreate()) //|| !latestNode.getGmtModified().equals(currentNode.getGmtModified()) || latestNode.getStatus() != currentNode.getStatus() || latestNode.getLogicalId() != currentNode.getLogicalId() || !latestNode.getPhysicalId().equals(currentNode.getPhysicalId()) || latestNode.getSerialId() != currentNode.getSerialId() || !latestNode.getIp().equals(currentNode.getIp()) || latestNode.getPort() != currentNode.getPort(); } private boolean isDifferent(StoreNode sn1, StoreNode sn2) { // ?? return (!sn1.getPhId().equals(sn2.getPhId())) || (sn1.getStatus() != sn2.getStatus()) || (sn1.getLogicId() != sn2.getLogicId()) || (sn1.getSequence() != sn2.getSequence()) || (!sn1.getIp().equals(sn2.getIp())) || (sn1.getPort() != sn2.getPort()); } public void start() { if (configLoadThread != null) { configLoadThread.start(); } } public void stop() { this.close(); } }