Java tutorial
/* * Copyright (c) 2013 Websquared, Inc. * All rights reserved. This program and the accompanying materials * are made available under the terms of the GNU Public License v2.0 * which accompanies this distribution, and is available at * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html * * Contributors: * swsong - initial API and implementation */ package org.fastcatsearch.ir; import java.io.File; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import javax.xml.bind.JAXBException; import org.apache.commons.io.FileUtils; import org.fastcatsearch.alert.ClusterAlertService; import org.fastcatsearch.cluster.NodeLoadBalancable; import org.fastcatsearch.common.QueryCacheModule; import org.fastcatsearch.control.JobService; import org.fastcatsearch.env.Environment; import org.fastcatsearch.exception.FastcatSearchException; import org.fastcatsearch.ir.analysis.AnalyzerFactoryManager; import org.fastcatsearch.ir.analysis.AnalyzerPoolManager; import org.fastcatsearch.ir.common.IRException; import org.fastcatsearch.ir.common.SettingException; import org.fastcatsearch.ir.config.CollectionConfig; import org.fastcatsearch.ir.config.CollectionContext; import org.fastcatsearch.ir.config.CollectionsConfig; import org.fastcatsearch.ir.config.CollectionsConfig.Collection; import org.fastcatsearch.ir.config.IndexingScheduleConfig; import org.fastcatsearch.ir.config.IndexingScheduleConfig.IndexingSchedule; import org.fastcatsearch.ir.config.JDBCSourceConfig; import org.fastcatsearch.ir.config.JDBCSourceInfo; import org.fastcatsearch.ir.config.JDBCSupportConfig; import org.fastcatsearch.ir.group.GroupResults; import org.fastcatsearch.ir.group.GroupsData; import org.fastcatsearch.ir.query.InternalSearchResult; import org.fastcatsearch.ir.query.Result; import org.fastcatsearch.ir.search.CollectionHandler; import org.fastcatsearch.ir.settings.AnalyzerSetting; import org.fastcatsearch.job.PriorityScheduledJob; import org.fastcatsearch.job.ScheduledJobEntry; import org.fastcatsearch.job.indexing.MasterCollectionAddIndexingJob; import org.fastcatsearch.job.indexing.MasterCollectionFullIndexingJob; import org.fastcatsearch.module.ModuleException; import org.fastcatsearch.notification.NotificationService; import org.fastcatsearch.notification.message.CollectionLoadErrorNotification; import org.fastcatsearch.service.AbstractService; import org.fastcatsearch.service.ServiceManager; import org.fastcatsearch.settings.SearchPageSettings; import org.fastcatsearch.settings.SettingFileNames; import org.fastcatsearch.settings.Settings; import org.fastcatsearch.util.CollectionContextUtil; import org.fastcatsearch.util.FilePaths; import org.fastcatsearch.util.JAXBConfigs; public class IRService extends AbstractService { private Map<String, CollectionHandler> collectionHandlerMap; // TODO ??? ?. private QueryCacheModule<String, Result> searchCache; private QueryCacheModule<String, InternalSearchResult> shardSearchCache; private QueryCacheModule<String, GroupResults> groupingCache; private QueryCacheModule<String, GroupsData> groupingDataCache; private QueryCacheModule<String, Result> documentCache; private CollectionsConfig collectionsConfig; private JDBCSourceConfig jdbcSourceConfig; private JDBCSupportConfig jdbcSupportConfig; private SearchPageSettings searchPageSettings; private File collectionsRoot; private RealtimeQueryCountModule realtimeQueryStatisticsModule; private AnalyzerFactoryManager analyzerFactoryManager; private Set<String> dataNodeCollectionIdSet; //? ??? . count ?. public IRService(Environment environment, Settings settings, ServiceManager serviceManager) { super(environment, settings, serviceManager); realtimeQueryStatisticsModule = new RealtimeQueryCountModule(environment, settings); } public void setAnalyzerFactoryManager(AnalyzerProvider analyzerProvider) { this.analyzerFactoryManager = analyzerProvider.getAnalyzerFactoryManager(); } protected boolean doStart() throws FastcatSearchException { try { realtimeQueryStatisticsModule.load(); } catch (Throwable t) { ClusterAlertService.getInstance().alert(t); } collectionHandlerMap = new ConcurrentHashMap<String, CollectionHandler>(); // collections ? ?. collectionsRoot = environment.filePaths().getCollectionsRoot().file(); try { collectionsConfig = JAXBConfigs.readConfig(new File(collectionsRoot, SettingFileNames.collections), CollectionsConfig.class); } catch (JAXBException e) { logger.error("[ERROR] fail to read collection config. " + e.getMessage(), e); ClusterAlertService.getInstance().alert(e); } try { jdbcSourceConfig = JAXBConfigs.readConfig(new File(collectionsRoot, SettingFileNames.jdbcSourceConfig), JDBCSourceConfig.class); } catch (JAXBException e) { logger.error("[ERROR] fail to read jdbc source list. " + e.getMessage(), e); ClusterAlertService.getInstance().alert(e); } if (jdbcSourceConfig == null) { jdbcSourceConfig = new JDBCSourceConfig(); } try { jdbcSupportConfig = JAXBConfigs.readConfig( new File(collectionsRoot, SettingFileNames.jdbcSupportConfig), JDBCSupportConfig.class); } catch (JAXBException e) { logger.error("[ERROR] fail to read jdbc support. " + e.getMessage(), e); ClusterAlertService.getInstance().alert(e); } if (jdbcSupportConfig == null) { jdbcSupportConfig = new JDBCSupportConfig(); } File file = environment.filePaths().configPath().file(SettingFileNames.searchPageSettings); if (file.exists()) { try { searchPageSettings = JAXBConfigs.readConfig(file, SearchPageSettings.class); } catch (JAXBException e) { logger.error("[ERROR] fail to read search page settings. " + e.getMessage(), e); ClusterAlertService.getInstance().alert(e); } } else { searchPageSettings = new SearchPageSettings(); } dataNodeCollectionIdSet = new HashSet<String>(); List<Collection> collectionList = collectionsConfig.getCollectionList(); for (int collectionInx = 0; collectionInx < collectionList.size(); collectionInx++) { Collection collection = collectionList.get(collectionInx); try { String collectionId = collection.getId(); loadCollectionHandler(collectionId, collection); } catch (Throwable e) { logger.error("[ERROR] " + e.getMessage(), e); } } try { //? xml ? . JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.collections), collectionsConfig, CollectionsConfig.class); } catch (JAXBException e) { logger.error("", e); ClusterAlertService.getInstance().alert(e); } searchCache = new QueryCacheModule<String, Result>(environment, settings); shardSearchCache = new QueryCacheModule<String, InternalSearchResult>(environment, settings); groupingCache = new QueryCacheModule<String, GroupResults>(environment, settings); groupingDataCache = new QueryCacheModule<String, GroupsData>(environment, settings); documentCache = new QueryCacheModule<String, Result>(environment, settings); try { searchCache.load(); shardSearchCache.load(); groupingCache.load(); groupingDataCache.load(); documentCache.load(); } catch (ModuleException e) { ClusterAlertService.getInstance().alert(e); throw new FastcatSearchException("ERR-00320"); } return true; } public CollectionHandler loadCollectionHandler(String collectionId) throws IRException, SettingException { return loadCollectionHandler(collectionId, null); } public CollectionHandler loadCollectionHandler(String collectionId, Collection collection) throws IRException, SettingException { Throwable t = null; try { realtimeQueryStatisticsModule.registerQueryCount(collectionId); CollectionContext collectionContext = null; CollectionHandler collectionHandler = null; logger.info("Load Collection [{}]", collectionId); if (collection == null) { for (Collection col : collectionsConfig.getCollectionList()) { if (col.getId().equalsIgnoreCase(collectionId)) { collection = col; break; } } } try { collectionContext = loadCollectionContext(collection); } catch (SettingException e) { logger.error("context " + collectionId); throw e; } if (collectionContext == null) { return null; } else { collectionHandler = new CollectionHandler(collectionContext, analyzerFactoryManager); collectionHandler.setQueryCounter(realtimeQueryStatisticsModule.getQueryCounter(collectionId)); if (collectionContext.collectionConfig().getDataNodeList() != null && collectionContext .collectionConfig().getDataNodeList().contains(environment.myNodeId())) { dataNodeCollectionIdSet.add(collectionId); } } collectionHandler.load(); /* * ? handler . */ CollectionHandler previousCollectionHandler = collectionHandlerMap.put(collectionId, collectionHandler); if (previousCollectionHandler != null) { try { previousCollectionHandler.close(); } catch (IOException e) { throw new IRException(e); } } return collectionHandler; } catch (IRException e) { t = e; throw e; } catch (SettingException e) { t = e; throw e; } finally { if (t != null) { ClusterAlertService.getInstance().alert(t); NotificationService notificationService = ServiceManager.getInstance() .getService(NotificationService.class); notificationService.sendNotification(new CollectionLoadErrorNotification(collection.getId(), t)); } } } public JDBCSourceConfig getJDBCSourceConfig() { return jdbcSourceConfig; } public JDBCSourceInfo getJDBCSourceInfo(String jdbcId) { List<JDBCSourceInfo> jdbcList = jdbcSourceConfig.getJdbcSourceInfoList(); for (JDBCSourceInfo jdbcInfo : jdbcList) { logger.trace("jdbc-id:{}", jdbcInfo.getId()); if (jdbcId.equals(jdbcInfo.getId())) { return jdbcInfo; } } return null; } public JDBCSupportConfig getJDBCSupportConfig() { return jdbcSupportConfig; } public void updateJDBCSourceConfig(JDBCSourceConfig jdbcSourceConfig) throws JAXBException { this.jdbcSourceConfig = jdbcSourceConfig; //? xml ? . JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.jdbcSourceConfig), jdbcSourceConfig, JDBCSourceConfig.class); } public CollectionHandler collectionHandler(String collectionId) { if (collectionHandlerMap != null && collectionHandlerMap.containsKey(collectionId)) { return collectionHandlerMap.get(collectionId); } return null; } public CollectionContext collectionContext(String collectionId) { CollectionHandler h = collectionHandler(collectionId); if (h != null) { return h.collectionContext(); } else { return null; } } public List<Collection> getCollectionList() { return collectionsConfig.getCollectionList(); } public CollectionHandler createCollection(String collectionId, CollectionConfig collectionConfig) throws IRException, SettingException { if (collectionsConfig.contains(collectionId)) { // ? . throw new SettingException("Collection id already exists. " + collectionId); } try { FilePaths collectionFilePaths = environment.filePaths().collectionFilePaths(collectionId); collectionFilePaths.file().mkdirs(); CollectionContext collectionContext = CollectionContextUtil.create(collectionConfig, collectionFilePaths); collectionsConfig.addCollection(collectionId); JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.collections), collectionsConfig, CollectionsConfig.class); CollectionHandler collectionHandler = new CollectionHandler(collectionContext, analyzerFactoryManager); collectionHandlerMap.put(collectionId, collectionHandler); realtimeQueryStatisticsModule.registerQueryCount(collectionId); return collectionHandler; } catch (IRException e) { throw e; } catch (Exception e) { logger.error("Error while create collection", e); throw new SettingException(e); } } public CollectionContext loadCollectionContext(Collection collection) throws SettingException { FilePaths collectionFilePaths = environment.filePaths().collectionFilePaths(collection.getId()); if (!collectionFilePaths.file().exists()) { // . logger.error("[{}] .", collection); return null; } return CollectionContextUtil.load(collection, collectionFilePaths); } public boolean removeCollection(String collectionId) throws SettingException { if (!collectionsConfig.contains(collectionId)) { return false; } else { try { CollectionHandler collectionHandler = collectionHandlerMap.remove(collectionId); if (collectionHandler != null) { collectionHandler.close(); } collectionsConfig.removeCollection(collectionId); JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.collections), collectionsConfig, CollectionsConfig.class); FilePaths collectionFilePaths = environment.filePaths().collectionFilePaths(collectionId); //FileUtils.deleteDirectory(collectionFilePaths.file()); if (collectionFilePaths.file().exists()) { FileUtils.forceDelete(collectionFilePaths.file()); } return true; } catch (Exception e) { logger.error("Error while remove collection", e); throw new SettingException(e); } } } // public CollectionHandler promoteCollection(CollectionHandler collectionHandler, String collectionId) throws IRException { // Exception ex = null; // try { // String collectionTmp = collectionHandler.collectionId(); // File prevFile = collectionHandler.indexFilePaths().file(); // File newFile = environment.filePaths().collectionFilePaths(collectionId).file(); // collectionHandlerMap.remove(collectionHandler.collectionId()); // collectionHandler.close(); // // collectionsConfig.removeCollection(collectionTmp); // collectionsConfig.addCollection(collectionId); // logger.trace("remove ok. {}", collectionTmp); // JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.collections), collectionsConfig, CollectionsConfig.class); // logger.trace("ok collection promoted {}->{}:{}", collectionTmp, collectionId, newFile.getAbsoluteFile()); // prevFile.renameTo(newFile); // logger.trace("rename ok. {}", newFile); // collectionHandler = loadCollectionHandler(collectionId); // logger.trace("load ok. {}", collectionId); // return collectionHandler; // } catch (IOException e) { ex = e; // } catch (JAXBException e) { ex = e; // } catch (SettingException e) { ex = e; // } finally { // // if(ex!=null) { // logger.error("",ex); // throw new IRException(ex); // } // } // return null; // } public CollectionHandler removeCollectionHandler(String collectionId) { realtimeQueryStatisticsModule.removeQueryCount(collectionId); return collectionHandlerMap.remove(collectionId); } public CollectionHandler putCollectionHandler(String collectionId, CollectionHandler collectionHandler) { //write config file if not exists if (!collectionHandlerMap.containsKey(collectionId)) { collectionsConfig.addCollection(collectionId); try { JAXBConfigs.writeConfig(new File(collectionsRoot, SettingFileNames.collections), collectionsConfig, CollectionsConfig.class); } catch (JAXBException e) { logger.error("", e); } } return collectionHandlerMap.put(collectionId, collectionHandler); } public CollectionHandler loadCollectionHandler(CollectionContext collectionContext) throws IRException, SettingException { return new CollectionHandler(collectionContext, analyzerFactoryManager).load(); } protected boolean doStop() throws FastcatSearchException { realtimeQueryStatisticsModule.unload(); Iterator<Entry<String, CollectionHandler>> iter = collectionHandlerMap.entrySet().iterator(); while (iter.hasNext()) { Entry<String, CollectionHandler> entry = iter.next(); try { CollectionHandler collectionHandler = entry.getValue(); if (collectionHandler != null) { collectionHandler.close(); } logger.info("Shutdown Collection [{}]", entry.getKey()); } catch (IOException e) { logger.error("[ERROR] " + e.getMessage(), e); throw new FastcatSearchException("IRService ??.", e); } } searchCache.unload(); shardSearchCache.unload(); groupingCache.unload(); groupingDataCache.unload(); documentCache.unload(); collectionHandlerMap.clear(); return true; } @Override protected boolean doClose() throws FastcatSearchException { collectionHandlerMap = null; realtimeQueryStatisticsModule = null; return true; } public QueryCacheModule<String, Result> searchCache() { return searchCache; } public QueryCacheModule<String, GroupResults> groupingCache() { return groupingCache; } public QueryCacheModule<String, Result> documentCache() { return documentCache; } public void registerLoadBanlancer(NodeLoadBalancable nodeLoadBalancable) { // ? collectionId node? ?. for (Collection collection : getCollectionList()) { String collectionId = collection.getId(); CollectionHandler collectionHandler = collectionHandlerMap.get(collectionId); if (collectionHandler == null) { continue; } List<String> dataNodeIdList = collectionHandler.collectionContext().collectionConfig() .getDataNodeList(); nodeLoadBalancable.updateLoadBalance(collectionId, dataNodeIdList); } } public RealtimeQueryCountModule queryCountModule() { return realtimeQueryStatisticsModule; } private SimpleDateFormat simpleDateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); private static String IndexingSchduleKey = "INDEXING-SCHEDULE-"; public boolean reloadSchedule(String collectionId) { CollectionContext collectionContext = collectionContext(collectionId); if (collectionContext == null) { return false; } String scheduleKey = IndexingSchduleKey + collectionId; JobService.getInstance().cancelSchedule(scheduleKey); IndexingScheduleConfig indexingScheduleConfig = collectionContext(collectionId).indexingScheduleConfig(); IndexingSchedule fullIndexingSchedule = indexingScheduleConfig.getFullIndexingSchedule(); IndexingSchedule addIndexingSchedule = indexingScheduleConfig.getAddIndexingSchedule(); List<ScheduledJobEntry> scheduledEntryList = new ArrayList<ScheduledJobEntry>(); if (fullIndexingSchedule != null) { MasterCollectionFullIndexingJob job = new MasterCollectionFullIndexingJob(); job.setArgs(collectionId); if (fullIndexingSchedule.isActive()) { String startTime = fullIndexingSchedule.getStart(); int periodInSecond = fullIndexingSchedule.getPeriodInSecond(); try { logger.debug("Load full indexing schdule {} : {}: {}", collectionId, startTime, periodInSecond); if (periodInSecond >= 0) { //? ? ?. scheduledEntryList.add(new ScheduledJobEntry(job, simpleDateFormat.parse(startTime), periodInSecond, true)); } } catch (ParseException e) { logger.error("[{}] Full Indexing schedule time parse error : {}", collectionId, startTime); return false; } } } if (addIndexingSchedule != null) { MasterCollectionAddIndexingJob job = new MasterCollectionAddIndexingJob(); job.setArgs(collectionId); if (addIndexingSchedule.isActive()) { String startTime = addIndexingSchedule.getStart(); int periodInSecond = addIndexingSchedule.getPeriodInSecond(); try { logger.debug("Load add indexing schdule {} : {}: {}", collectionId, startTime, periodInSecond); if (periodInSecond >= 0) { scheduledEntryList.add(new ScheduledJobEntry(job, simpleDateFormat.parse(startTime), periodInSecond, false)); } } catch (ParseException e) { logger.error("[{}] Add Indexing schedule time parse error : {}", collectionId, startTime); return false; } } } if (scheduledEntryList.size() > 0) { PriorityScheduledJob scheduledJob = new PriorityScheduledJob(scheduleKey, scheduledEntryList); JobService.getInstance().schedule(scheduledJob, true); } else { logger.info("Collection {} has no indexing schedule.", collectionId); } return true; } public void reloadAllSchedule() { // ? ?. for (CollectionsConfig.Collection collection : getCollectionList()) { String collectionId = collection.getId(); reloadSchedule(collectionId); } } public Set<String> getDataNodeCollectionIdSet() { return dataNodeCollectionIdSet; } // ? ? . public List<String> getSearchNodeList() { Set<String> searchNodeSet = new HashSet<String>(); for (CollectionHandler collectionHandler : collectionHandlerMap.values()) { List<String> searchNodeList = collectionHandler.collectionContext().collectionConfig() .getSearchNodeList(); if (searchNodeList != null) { for (String searchNodeId : searchNodeList) { searchNodeSet.add(searchNodeId); } } } return new ArrayList<String>(searchNodeSet); } public AnalyzerPoolManager createAnalyzerPoolManager(List<AnalyzerSetting> analyzerSettingList) { AnalyzerPoolManager analyzerPoolManager = new AnalyzerPoolManager(); analyzerPoolManager.register(analyzerSettingList, analyzerFactoryManager); return analyzerPoolManager; } public SearchPageSettings getSearchPageSettings() { return searchPageSettings; } public void updateSearchPageSettings(SearchPageSettings searchPageSettings) { this.searchPageSettings = searchPageSettings; } }