Java tutorial
/* * #%L * Alfresco Solr 4 * %% * Copyright (C) 2005 - 2016 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of * the paid license agreement will prevail. Otherwise, the software is * provided under the following open source license terms: * * Alfresco is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Alfresco is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with Alfresco. If not, see <http://www.gnu.org/licenses/>. * #L% */ package org.alfresco.solr.tracker; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Properties; import java.util.concurrent.ConcurrentLinkedQueue; import org.alfresco.error.AlfrescoRuntimeException; import org.alfresco.httpclient.AuthenticationException; import org.alfresco.repo.index.shard.ShardMethodEnum; import org.alfresco.repo.index.shard.ShardState; import org.alfresco.repo.index.shard.ShardStateBuilder; import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.BoundedDeque; import org.alfresco.solr.InformationServer; import org.alfresco.solr.NodeReport; import org.alfresco.solr.TrackerState; import org.alfresco.solr.adapters.IOpenBitSet; import org.alfresco.solr.client.GetNodesParameters; import org.alfresco.solr.client.Node; import org.alfresco.solr.client.Node.SolrApiNodeStatus; import org.alfresco.solr.client.SOLRAPIClient; import org.alfresco.solr.client.Transaction; import org.alfresco.solr.client.Transactions; import org.apache.commons.codec.EncoderException; import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /* * This tracks two things: transactions and metadata nodes * @author Ahmed Owian */ public class MetadataTracker extends AbstractTracker implements Tracker { protected final static Logger log = LoggerFactory.getLogger(MetadataTracker.class); private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 100; private static final int DEFAULT_NODE_BATCH_SIZE = 10; private int transactionDocsBatchSize = DEFAULT_TRANSACTION_DOCS_BATCH_SIZE; private int nodeBatchSize = DEFAULT_NODE_BATCH_SIZE; private ConcurrentLinkedQueue<Long> transactionsToReindex = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<Long> transactionsToIndex = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<Long> transactionsToPurge = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<Long> nodesToReindex = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<Long> nodesToIndex = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<Long>(); private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<String>(); public MetadataTracker(Properties p, SOLRAPIClient client, String coreName, InformationServer informationServer) { super(p, client, coreName, informationServer); transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100")); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10")); threadHandler = new ThreadHandler(p, coreName, "MetadataTracker"); } MetadataTracker() { // Testing purposes only } @Override protected void doTrack() throws AuthenticationException, IOException, JSONException, EncoderException { // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); if (modelTracker != null && modelTracker.hasModels()) { purgeTransactions(); purgeNodes(); reindexTransactions(); reindexNodes(); reindexNodesByQuery(); indexTransactions(); indexNodes(); trackRepository(); } } private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException { // Is the InformationServer ready to update int registeredSearcherCount = this.infoSrv.getRegisteredSearcherCount(); if (registeredSearcherCount >= getMaxLiveSearchers()) { log.info(".... skipping tracking registered searcher count = " + registeredSearcherCount); return; } checkShutdown(); if (!isMaster && isSlave) { // Dynamic registration ShardState shardstate = getShardState(); client.getTransactions(0L, null, 0L, null, 0, shardstate); return; } // Check we are tracking the correct repository TrackerState state = super.getTrackerState(); checkRepoAndIndexConsistency(state); checkShutdown(); trackTransactions(); } /** * @param state * @return */ private ShardState getShardState() { TrackerState state = super.getTrackerState(); ShardState shardstate = ShardStateBuilder.shardState().withMaster(isMaster) .withLastUpdated(System.currentTimeMillis()) .withLastIndexedChangeSetCommitTime(state.getLastIndexedChangeSetCommitTime()) .withLastIndexedChangeSetId(state.getLastIndexedChangeSetId()) .withLastIndexedTxCommitTime(state.getLastIndexedTxCommitTime()) .withLastIndexedTxId(state.getLastIndexedTxId()).withShardInstance() .withBaseUrl(infoSrv.getBaseUrl()).withPort(infoSrv.getPort()).withHostName(infoSrv.getHostName()) .withShard().withInstance(shardInstance).withFloc().withNumberOfShards(shardCount) .withAddedStoreRef(storeRef).withTemplate(shardTemplate).withHasContent(transformContent) .withShardMethod(ShardMethodEnum.MOD_ACL_ID).endFloc().endShard().endShardInstance().build(); return shardstate; } /** * Checks the first and last TX time * @param state the state of this tracker * @throws AuthenticationException * @throws IOException * @throws JSONException */ private void checkRepoAndIndexConsistency(TrackerState state) throws AuthenticationException, IOException, JSONException { Transactions firstTransactions = null; if (state.getLastGoodTxCommitTimeInIndex() == 0) { state.setCheckedLastTransactionTime(true); state.setCheckedFirstTransactionTime(true); log.info("No transactions found - no verification required"); firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); long firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); state.setLastGoodTxCommitTimeInIndex(firstTransactionCommitTime); setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); } } if (!state.isCheckedFirstTransactionTime()) { firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); long firstTxId = firstTransaction.getId(); long firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); int setSize = this.infoSrv.getTxDocsSize("" + firstTxId, "" + firstTransactionCommitTime); if (setSize == 0) { log.error("First transaction was not found with the correct timestamp."); log.error( "SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); log.error( "If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); log.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp"); } else if (setSize == 1) { state.setCheckedFirstTransactionTime(true); log.info("Verified first transaction and timestamp in index"); } else { log.warn("Duplicate initial transaction found with correct timestamp"); } } } // Checks that the last TxId in solr is <= last TxId in repo if (!state.isCheckedLastTransactionTime()) { if (firstTransactions == null) { firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1); } setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state); Long maxTxnCommitTimeInRepo = firstTransactions.getMaxTxnCommitTime(); Long maxTxnIdInRepo = firstTransactions.getMaxTxnId(); if (maxTxnCommitTimeInRepo != null && maxTxnIdInRepo != null) { Transaction maxTxInIndex = this.infoSrv.getMaxTransactionIdAndCommitTimeInIndex(); if (maxTxInIndex.getCommitTimeMs() > maxTxnCommitTimeInRepo) { log.error("Last transaction was found in index with timestamp later than that of repository."); log.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); log.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + maxTxnCommitTimeInRepo); log.error( "SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); log.error( "If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); log.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp"); } else { state.setCheckedLastTransactionTime(true); log.info( "Verified last transaction timestamp in index less than or equal to that of repository."); } } } } private void indexTransactions() throws IOException, AuthenticationException, JSONException { long startElapsed = System.nanoTime(); int docCount = 0; boolean requiresCommit = false; while (transactionsToIndex.peek() != null) { Long transactionId = transactionsToIndex.poll(); if (transactionId != null) { Transactions transactions = client.getTransactions(null, transactionId, null, transactionId + 1, 1); if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId()))) { Transaction info = transactions.getTransactions().get(0); GetNodesParameters gnp = new GetNodesParameters(); ArrayList<Long> txs = new ArrayList<Long>(); txs.add(info.getId()); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); List<Node> nodes = client.getNodes(gnp, (int) info.getUpdates()); for (Node node : nodes) { docCount++; if (log.isDebugEnabled()) { log.debug(node.toString()); } this.infoSrv.indexNode(node, false); checkShutdown(); } // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, false); requiresCommit = true; trackerStats.addTxDocs(nodes.size()); } } if (docCount > batchCount) { if (this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { checkShutdown(); this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); startElapsed = endElapsed; docCount = 0; requiresCommit = false; } } } if (requiresCommit || (docCount > 0)) { checkShutdown(); this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); } } private void indexNodes() throws IOException, AuthenticationException, JSONException { boolean requiresCommit = false; while (nodesToIndex.peek() != null) { Long nodeId = nodesToIndex.poll(); if (nodeId != null) { Node node = new Node(); node.setId(nodeId); node.setStatus(SolrApiNodeStatus.UNKNOWN); node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, false); requiresCommit = true; } checkShutdown(); } if (requiresCommit) { checkShutdown(); this.infoSrv.commit(); } } private void reindexTransactions() throws IOException, AuthenticationException, JSONException { long startElapsed = System.nanoTime(); int docCount = 0; boolean requiresCommit = false; while (transactionsToReindex.peek() != null) { Long transactionId = transactionsToReindex.poll(); if (transactionId != null) { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByTransactionId(transactionId); Transactions transactions = client.getTransactions(null, transactionId, null, transactionId + 1, 1); if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId()))) { Transaction info = transactions.getTransactions().get(0); GetNodesParameters gnp = new GetNodesParameters(); ArrayList<Long> txs = new ArrayList<Long>(); txs.add(info.getId()); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); List<Node> nodes = client.getNodes(gnp, (int) info.getUpdates()); for (Node node : nodes) { docCount++; if (log.isDebugEnabled()) { log.debug(node.toString()); } this.infoSrv.indexNode(node, true); checkShutdown(); } // Index the transaction doc after the node - if this is not found then a reindex will be done. this.infoSrv.indexTransaction(info, true); requiresCommit = true; } } if (docCount > batchCount) { if (this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { checkShutdown(); this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); startElapsed = endElapsed; docCount = 0; requiresCommit = false; } } } if (requiresCommit || (docCount > 0)) { checkShutdown(); this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); } } private void reindexNodes() throws IOException, AuthenticationException, JSONException { boolean requiresCommit = false; while (nodesToReindex.peek() != null) { Long nodeId = nodesToReindex.poll(); if (nodeId != null) { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByNodeId(nodeId); Node node = new Node(); node.setId(nodeId); node.setStatus(SolrApiNodeStatus.UNKNOWN); node.setTxnId(Long.MAX_VALUE); this.infoSrv.indexNode(node, true); requiresCommit = true; } checkShutdown(); } if (requiresCommit) { checkShutdown(); this.infoSrv.commit(); } } private void reindexNodesByQuery() throws IOException, AuthenticationException, JSONException { boolean requiresCommit = false; while (queriesToReindex.peek() != null) { String query = queriesToReindex.poll(); if (query != null) { this.infoSrv.reindexNodeByQuery(query); requiresCommit = true; } checkShutdown(); } if (requiresCommit) { checkShutdown(); this.infoSrv.commit(); } } private void purgeTransactions() throws IOException, AuthenticationException, JSONException { boolean requiresCommit = false; while (transactionsToPurge.peek() != null) { Long transactionId = transactionsToPurge.poll(); if (transactionId != null) { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByTransactionId(transactionId); requiresCommit = true; } checkShutdown(); } if (requiresCommit) { checkShutdown(); this.infoSrv.commit(); } } private void purgeNodes() throws IOException, AuthenticationException, JSONException { boolean requiresCommit = false; while (nodesToPurge.peek() != null) { Long nodeId = nodesToPurge.poll(); if (nodeId != null) { // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByNodeId(nodeId); requiresCommit = true; } checkShutdown(); } if (requiresCommit) { checkShutdown(); this.infoSrv.commit(); } } protected Long getTxFromCommitTime(BoundedDeque<Transaction> txnsFound, long lastGoodTxCommitTimeInIndex) { if (txnsFound.size() > 0) { return txnsFound.getLast().getCommitTimeMs(); } else { return lastGoodTxCommitTimeInIndex; } } private boolean alreadyFoundTransactions(BoundedDeque<Transaction> txnsFound, Transactions transactions) { if (txnsFound.size() == 0) { return false; } if (transactions.getTransactions().size() == 1) { return transactions.getTransactions().get(0).getId() == txnsFound.getLast().getId(); } else { HashSet<Transaction> alreadyFound = new HashSet<Transaction>(txnsFound.getDeque()); for (Transaction txn : transactions.getTransactions()) { if (!alreadyFound.contains(txn)) { return false; } } return true; } } protected Transactions getSomeTransactions(BoundedDeque<Transaction> txnsFound, Long fromCommitTime, long timeStep, int maxResults, long endTime) throws AuthenticationException, IOException, JSONException, EncoderException { long actualTimeStep = timeStep; ShardState shardstate = getShardState(); Transactions transactions; // step forward in time until we find something or hit the time bound // max id unbounded Long startTime = fromCommitTime == null ? Long.valueOf(0L) : fromCommitTime; do { transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate); startTime += actualTimeStep; //actualTimeStep *= 2; if (actualTimeStep > TIME_STEP_32_DAYS_IN_MS) { actualTimeStep = TIME_STEP_32_DAYS_IN_MS; } } while (((transactions.getTransactions().size() == 0) && (startTime < endTime)) || ((transactions.getTransactions().size() > 0) && alreadyFoundTransactions(txnsFound, transactions))); return transactions; } protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException { long startElapsed = System.nanoTime(); boolean indexed = false; boolean upToDate = false; Transactions transactions; BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100); HashSet<Transaction> txsIndexed = new LinkedHashSet<>(); TrackerState state = this.getTrackerState(); long totalUpdatedDocs = 0; int docCount = 0; do { Long fromCommitTime = getTxFromCommitTime(txnsFound, state.getLastGoodTxCommitTimeInIndex()); transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, state.getTimeToStopIndexing()); setLastTxCommitTimeAndTxIdInTrackerState(transactions, state); log.info("Scanning transactions ..."); if (transactions.getTransactions().size() > 0) { log.info(".... from " + transactions.getTransactions().get(0)); log.info( ".... to " + transactions.getTransactions().get(transactions.getTransactions().size() - 1)); } else { log.info(".... none found after lastTxCommitTime " + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime())); } ArrayList<Transaction> txBatch = new ArrayList<>(); for (Transaction info : transactions.getTransactions()) { boolean isInIndex = (info.getCommitTimeMs() <= state.getLastIndexedTxCommitTime()) && this.infoSrv.isInIndex(AlfrescoSolrDataModel.getTransactionDocumentId(info.getId())); if (isInIndex) { txnsFound.add(info); } else { // Make sure we do not go ahead of where we started - we will check the holes here // correctly next time if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { upToDate = true; break; } txBatch.add(info); if (getUpdateAndDeleteCount(txBatch) > this.transactionDocsBatchSize) { indexed = true; docCount += indexBatchOfTransactions(txBatch); for (Transaction scheduledTx : txBatch) { txnsFound.add(scheduledTx); txsIndexed.add(scheduledTx); } txBatch.clear(); } } if (docCount > batchCount) { if (super.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) { indexTransactionsAfterAsynchronous(txsIndexed, state); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); startElapsed = endElapsed; docCount = 0; } } checkShutdown(); } if (!txBatch.isEmpty()) { indexed = true; if (this.getUpdateAndDeleteCount(txBatch) > 0) { docCount += indexBatchOfTransactions(txBatch); } for (Transaction scheduledTx : txBatch) { txnsFound.add(scheduledTx); txsIndexed.add(scheduledTx); } txBatch.clear(); } totalUpdatedDocs += docCount; } while ((transactions.getTransactions().size() > 0) && (upToDate == false)); log.info("total number of docs with metadata updated: " + totalUpdatedDocs); if (indexed) { indexTransactionsAfterAsynchronous(txsIndexed, state); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); } } private void setLastTxCommitTimeAndTxIdInTrackerState(Transactions transactions, TrackerState state) { Long maxTxnCommitTime = transactions.getMaxTxnCommitTime(); if (maxTxnCommitTime != null) { state.setLastTxCommitTimeOnServer(maxTxnCommitTime); } Long maxTxnId = transactions.getMaxTxnId(); if (maxTxnId != null) { state.setLastTxIdOnServer(maxTxnId); } } private void indexTransactionsAfterAsynchronous(HashSet<Transaction> txsIndexed, TrackerState state) throws IOException { waitForAsynchronous(); for (Transaction tx : txsIndexed) { super.infoSrv.indexTransaction(tx, true); // Transactions are ordered by commit time and tie-broken by tx id if (tx.getCommitTimeMs() > state.getLastIndexedTxCommitTime() || tx.getCommitTimeMs() == state.getLastIndexedTxCommitTime() && tx.getId() > state.getLastIndexedTxId()) { state.setLastIndexedTxCommitTime(tx.getCommitTimeMs()); state.setLastIndexedTxId(tx.getId()); } trackerStats.addTxDocs((int) (tx.getDeletes() + tx.getUpdates())); } txsIndexed.clear(); super.infoSrv.commit(); } private long getUpdateAndDeleteCount(List<Transaction> txs) { long count = 0; for (Transaction tx : txs) { count += (tx.getUpdates() + tx.getDeletes()); } return count; } private int indexBatchOfTransactions(List<Transaction> txBatch) throws AuthenticationException, IOException, JSONException { int nodeCount = 0; ArrayList<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size()); GetNodesParameters gnp = new GetNodesParameters(); ArrayList<Long> txIds = new ArrayList<Long>(); for (Transaction tx : txBatch) { if (tx.getUpdates() > 0 || tx.getDeletes() > 0) { nonEmptyTxs.add(tx); txIds.add(tx.getId()); } } gnp.setTransactionIds(txIds); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); List<Node> nodes = client.getNodes(gnp, Integer.MAX_VALUE); ArrayList<Node> nodeBatch = new ArrayList<>(); for (Node node : nodes) { if (log.isDebugEnabled()) { log.debug(node.toString()); } nodeBatch.add(node); if (nodeBatch.size() > nodeBatchSize) { nodeCount += nodeBatch.size(); NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); this.threadHandler.scheduleTask(niwr); nodeBatch = new ArrayList<>(); } } if (nodeBatch.size() > 0) { nodeCount += nodeBatch.size(); NodeIndexWorkerRunnable niwr = new NodeIndexWorkerRunnable(this.threadHandler, nodeBatch, this.infoSrv); this.threadHandler.scheduleTask(niwr); nodeBatch = new ArrayList<>(); } return nodeCount; } class NodeIndexWorkerRunnable extends AbstractWorkerRunnable { InformationServer infoServer; List<Node> nodes; NodeIndexWorkerRunnable(QueueHandler queueHandler, List<Node> nodes, InformationServer infoServer) { super(queueHandler); this.infoServer = infoServer; this.nodes = nodes; } @Override protected void doWork() throws IOException, AuthenticationException, JSONException { List<Node> filteredNodes = filterNodes(nodes); if (filteredNodes.size() > 0) { this.infoServer.indexNodes(filteredNodes, true); } } private List<Node> filterNodes(List<Node> nodes) { ArrayList<Node> filteredList = new ArrayList<Node>(nodes.size()); for (Node node : nodes) { if (isInAclShard(node.getAclId())) { filteredList.add(node); } else { // Cascade update children of this node if they are in this shard if (node.getStatus() == SolrApiNodeStatus.UPDATED) { Node doCascade = new Node(); doCascade.setAclId(node.getAclId()); doCascade.setId(node.getId()); doCascade.setNodeRef(node.getNodeRef()); doCascade.setStatus(SolrApiNodeStatus.NON_SHARD_UPDATED); doCascade.setTenant(node.getTenant()); doCascade.setTxnId(node.getTxnId()); filteredList.add(doCascade); } else // DELETED & UNKNOWN { // Make sure anything no longer relevant to this shard is deleted. Node doDelete = new Node(); doDelete.setAclId(node.getAclId()); doDelete.setId(node.getId()); doDelete.setNodeRef(node.getNodeRef()); doDelete.setStatus(SolrApiNodeStatus.NON_SHARD_DELETED); doDelete.setTenant(node.getTenant()); doDelete.setTxnId(node.getTxnId()); filteredList.add(doDelete); } } } return filteredList; } } public NodeReport checkNode(Long dbid) { NodeReport nodeReport = new NodeReport(); nodeReport.setDbid(dbid); // In DB GetNodesParameters parameters = new GetNodesParameters(); parameters.setFromNodeId(dbid); parameters.setToNodeId(dbid); List<Node> dbnodes; try { dbnodes = client.getNodes(parameters, 1); if (dbnodes.size() == 1) { Node dbnode = dbnodes.get(0); nodeReport.setDbNodeStatus(dbnode.getStatus()); nodeReport.setDbTx(dbnode.getTxnId()); } else { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); nodeReport.setDbTx(-1l); } } catch (IOException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); nodeReport.setDbTx(-2l); } catch (JSONException e) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); nodeReport.setDbTx(-3l); } catch (AuthenticationException e1) { nodeReport.setDbNodeStatus(SolrApiNodeStatus.UNKNOWN); nodeReport.setDbTx(-4l); } this.infoSrv.addCommonNodeReportInfo(nodeReport); return nodeReport; } public NodeReport checkNode(Node node) { NodeReport nodeReport = new NodeReport(); nodeReport.setDbid(node.getId()); nodeReport.setDbNodeStatus(node.getStatus()); nodeReport.setDbTx(node.getTxnId()); this.infoSrv.addCommonNodeReportInfo(nodeReport); return nodeReport; } public List<Node> getFullNodesForDbTransaction(Long txid) { try { GetNodesParameters gnp = new GetNodesParameters(); ArrayList<Long> txs = new ArrayList<Long>(); txs.add(txid); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); return client.getNodes(gnp, Integer.MAX_VALUE); } catch (IOException e) { throw new AlfrescoRuntimeException("Failed to get nodes", e); } catch (JSONException e) { throw new AlfrescoRuntimeException("Failed to get nodes", e); } catch (AuthenticationException e) { throw new AlfrescoRuntimeException("Failed to get nodes", e); } } public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) throws IOException, AuthenticationException, JSONException, EncoderException { // DB TX Count long firstTransactionCommitTime = 0; Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000L, 1); if (firstTransactions.getTransactions().size() > 0) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); } IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance(); Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime); if (fromTime != null) { lastTxCommitTime = fromTime; } long maxTxId = 0; Long minTxId = null; Transactions transactions; BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100); long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); DO: do { transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, endTime); for (Transaction info : transactions.getTransactions()) { // include if (toTime != null) { if (info.getCommitTimeMs() > toTime.longValue()) { break DO; } } if (toTx != null) { if (info.getId() > toTx.longValue()) { break DO; } } // bounds for later loops if (minTxId == null) { minTxId = info.getId(); } if (maxTxId < info.getId()) { maxTxId = info.getId(); } lastTxCommitTime = info.getCommitTimeMs(); txIdsInDb.set(info.getId()); txnsFound.add(info); } } while (transactions.getTransactions().size() > 0); return this.infoSrv.reportIndexTransactions(minTxId, txIdsInDb, maxTxId); } public void addTransactionToPurge(Long txId) { this.transactionsToPurge.offer(txId); } public void addNodeToPurge(Long nodeId) { this.nodesToPurge.offer(nodeId); } public void addTransactionToReindex(Long txId) { this.transactionsToReindex.offer(txId); } public void addNodeToReindex(Long nodeId) { nodesToReindex.offer(nodeId); } public void addTransactionToIndex(Long txId) { transactionsToIndex.offer(txId); } public void addNodeToIndex(Long nodeId) { this.nodesToIndex.offer(nodeId); } /** * @param query */ public void addQueryToReindex(String query) { this.queriesToReindex.offer(query); } }