Java tutorial
/******************************************************************************* * Copyright (c) 2013 aegif. * * This file is part of NemakiWare. * * NemakiWare is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * NemakiWare is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along with NemakiWare. * If not, see <http://www.gnu.org/licenses/>. * * Contributors: * linzhixing(https://github.com/linzhixing) - initial API and implementation ******************************************************************************/ package jp.aegif.nemaki.tracker; import static org.apache.solr.handler.extraction.ExtractingParams.UNKNOWN_FIELD_PREFIX; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.Set; import jp.aegif.nemaki.NemakiCoreAdminHandler; import jp.aegif.nemaki.util.CmisSessionFactory; import jp.aegif.nemaki.util.Constant; import jp.aegif.nemaki.util.PropertyKey; import jp.aegif.nemaki.util.PropertyManager; import jp.aegif.nemaki.util.StringPool; import jp.aegif.nemaki.util.NemakiTokenManager; import jp.aegif.nemaki.util.impl.PropertyManagerImpl; import jp.aegif.nemaki.util.yaml.RepositorySettings; import org.apache.chemistry.opencmis.client.api.ChangeEvent; import org.apache.chemistry.opencmis.client.api.ChangeEvents; import org.apache.chemistry.opencmis.client.api.Session; import org.apache.chemistry.opencmis.commons.exceptions.CmisRuntimeException; import org.apache.chemistry.opencmis.commons.spi.CmisBinding; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.core.CloseHook; import org.apache.solr.core.SolrCore; /** * Index tracking class * * @author linzhixing * */ public class CoreTracker extends CloseHook { private static final Object LOCK = new Object(); Logger logger = Logger.getLogger(CoreTracker.class); NemakiCoreAdminHandler adminHandler; SolrCore core; SolrServer indexServer; SolrServer tokenServer; CmisBinding cmisBinding; NemakiTokenManager nemakiTokenManager; public CoreTracker(NemakiCoreAdminHandler adminHandler, SolrCore core, SolrServer indexServer, SolrServer tokenServer) { super(); this.adminHandler = adminHandler; this.core = core; this.indexServer = indexServer; this.tokenServer = tokenServer; this.nemakiTokenManager = new NemakiTokenManager(); } public SolrServer getIndexServer() { return indexServer; } @Override public void preClose(SolrCore core) { } @Override public void postClose(SolrCore core) { } /** * Initialize a specified Solr core */ public void initCore() { synchronized (LOCK) { try { // Initialize all documents indexServer.deleteByQuery("*:*"); indexServer.commit(); logger.info(core.getName() + ":Successfully initialized!"); tokenServer.deleteByQuery("*:*"); tokenServer.commit(); logger.info(core.getName() + ":Successfully initialized!"); } catch (SolrServerException e) { logger.error(core.getName() + ":Initialization failed!", e); } catch (IOException e) { logger.error(core.getName() + ":Initialization failed!", e); } } } public void initCore(String repositoryId) { synchronized (LOCK) { try { // Initialize all documents indexServer.deleteByQuery(Constant.FIELD_REPOSITORY_ID + ":" + repositoryId); indexServer.commit(); logger.info(core.getName() + ":Successfully initialized!"); storeLatestChangeToken("", repositoryId); } catch (SolrServerException e) { logger.error(core.getName() + ":Initialization failed!", e); } catch (IOException e) { logger.error(core.getName() + ":Initialization failed!", e); } } } /** * Read CMIS change logs and Index them * * @param trackingType */ public void index(String trackingType) { RepositorySettings settings = CmisSessionFactory.getRepositorySettings(); for (String repositoryId : settings.getIds()) { index(trackingType, repositoryId); // TODO multi-threding } } public void index(String trackingType, String repositoryId) { synchronized (LOCK) { ChangeEvents changeEvents = getCmisChangeLog(trackingType, repositoryId); if (changeEvents == null) { return; } List<ChangeEvent> events = changeEvents.getChangeEvents(); // After 2nd crawling, discard the first item // Because the specs say that it's included in the results String token = readLatestChangeToken(repositoryId); if (!StringUtils.isEmpty(token)) { if (!org.apache.commons.collections.CollectionUtils.isEmpty(events)) { events.remove(0); } } if (events.isEmpty()) return; // Parse filtering configuration PropertyManager pm = new PropertyManagerImpl(StringPool.PROPERTIES_NAME); boolean fulltextEnabled = Boolean.TRUE.toString() .equalsIgnoreCase(pm.readValue(PropertyKey.SOLR_TRACKING_FULLTEXT_ENABLED)); boolean mimeTypeFilterEnabled = false; // default List<String> allowedMimeTypeFilter = new ArrayList<String>(); // default if (fulltextEnabled) { String _filter = pm.readValue(PropertyKey.SOLR_TRACKING_MIMETYPE_FILTER_ENABLED); mimeTypeFilterEnabled = Boolean.TRUE.toString().equalsIgnoreCase(_filter); if (mimeTypeFilterEnabled) { allowedMimeTypeFilter = pm.readValues(PropertyKey.SOLR_TRACKING_MIMETYPE); } } // Extract only the last events of each objectId List<ChangeEvent> list = extractChangeEvent(events); PropertyManager propMgr = new PropertyManagerImpl(StringPool.PROPERTIES_NAME); int numberOfThread = Integer.valueOf(propMgr.readValue(PropertyKey.SOLR_TRACKING_NUMBER_OF_THREAD)); int numberPerThread = list.size() / numberOfThread; if (list.size() < numberOfThread) { numberOfThread = list.size(); numberPerThread = 1; } for (int i = 0; i <= numberOfThread; i++) { int toIndex = (numberPerThread * (i + 1) > list.size()) ? list.size() : numberPerThread * (i + 1); List<ChangeEvent> listPerThread = list.subList(numberPerThread * i, toIndex); Session cmisSession = CmisSessionFactory.getSession(repositoryId); Registration registration = new Registration(cmisSession, core, indexServer, listPerThread, fulltextEnabled, mimeTypeFilterEnabled, allowedMimeTypeFilter); Thread t = new Thread(registration); t.start(); try { t.join(); } catch (InterruptedException e) { logger.error(e); } } // Save the latest token storeLatestChangeToken(changeEvents.getLatestChangeLogToken(), repositoryId); // In case of FUll mode, repeat until indexing all change logs if (Constant.MODE_FULL.equals(trackingType)) { index(Constant.MODE_FULL, repositoryId); } } } /** * Get the last change token stored in Solr * * @return */ private String readLatestChangeToken(String repositoryId) { SolrQuery solrQuery = new SolrQuery(); solrQuery.setQuery(Constant.FIELD_REPOSITORY_ID + ":" + repositoryId); QueryResponse resp = null; try { resp = tokenServer.query(solrQuery); } catch (SolrServerException e) { e.printStackTrace(); } String latestChangeToken = ""; if (resp != null && resp.getResults() != null && resp.getResults().getNumFound() != 0) { SolrDocument doc = resp.getResults().get(0); latestChangeToken = (String) doc.get(Constant.FIELD_TOKEN); } else { logger.info("No latest change token found for repository: " + repositoryId); logger.info("Set blank latest change token for repository: " + repositoryId); storeLatestChangeToken("", repositoryId); } return latestChangeToken; } /** * Store the last change token in Solr * * @return */ private void storeLatestChangeToken(String token, String repositoryId) { Map<String, Object> map = new HashMap<String, Object>(); map.put(Constant.FIELD_REPOSITORY_ID, repositoryId); map.put(Constant.FIELD_TOKEN, token); AbstractUpdateRequest req = buildUpdateRequest(map); try { tokenServer.request(req); } catch (SolrServerException e) { logger.error("Failed to store latest change token in Solr!", e); } catch (IOException e) { logger.error("Failed to store latest change token in Solr!", e); } } /** * Get CMIS change logs * * @param trackingType * @return */ private ChangeEvents getCmisChangeLog(String trackingType, String repositoryId) { PropertyManager propMgr = new PropertyManagerImpl(StringPool.PROPERTIES_NAME); String _latestToken = readLatestChangeToken(repositoryId); String latestToken = (StringUtils.isEmpty(_latestToken)) ? null : _latestToken; long _numItems = 0; if (Constant.MODE_DELTA.equals(trackingType)) { _numItems = Long.valueOf(propMgr.readValue(PropertyKey.CMIS_CHANGELOG_ITEMS_DELTA)); } else if (Constant.MODE_FULL.equals(trackingType)) { _numItems = Long.valueOf(propMgr.readValue(PropertyKey.CMIS_CHANGELOG_ITEMS_FULL)); } long numItems = (-1 == _numItems) ? Long.MAX_VALUE : Long.valueOf(_numItems); Session cmisSession = CmisSessionFactory.getSession(repositoryId); if (cmisSession == null) { return null; } try { // No need for Sorting // (Specification requires they are returned by ASCENDING) return cmisSession.getContentChanges(latestToken, false, numItems); } catch (CmisRuntimeException ex) { // On error reset session. CmisSessionFactory.clearSession(repositoryId); throw ex; } } /** * * @param events * @return */ private List<ChangeEvent> extractChangeEvent(List<ChangeEvent> events) { List<ChangeEvent> list = new ArrayList<ChangeEvent>(); Set<String> objectIds = new HashSet<String>(); int size = events.size(); ListIterator<ChangeEvent> iterator = events.listIterator(size); while (iterator.hasPrevious()) { ChangeEvent event = iterator.previous(); if (objectIds.contains(event.getObjectId())) { continue; } else { objectIds.add(event.getObjectId()); list.add(event); } } Collections.reverse(list); return list; } /** * Build an update request to Solr without file * * @param content * @return */ // TODO Unify that of Registration class private AbstractUpdateRequest buildUpdateRequest(Map<String, Object> map) { UpdateRequest up = new UpdateRequest(); SolrInputDocument sid = new SolrInputDocument(); // Set SolrDocument parameters Iterator<String> iterator = map.keySet().iterator(); while (iterator.hasNext()) { String key = iterator.next(); sid.addField(key, map.get(key)); } // Set UpdateRequest up.add(sid); // Ignored(for schema.xml, ignoring some SolrCell meta fields) up.setParam(UNKNOWN_FIELD_PREFIX, "ignored_"); // Set Solr action parameter up.setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true); return up; } }