List of usage examples for java.util TreeSet pollFirst
public E pollFirst()
From source file:org.apache.solr.client.solrj.io.eval.KnnEvaluator.java
@Override public Object doWork(Object... values) throws IOException { if (values.length < 3) { throw new IOException("knn expects three parameters a Matrix, numeric array and k"); }// w w w. ja v a 2s.c om Matrix matrix = null; double[] vec = null; int k = 0; if (values[0] instanceof Matrix) { matrix = (Matrix) values[0]; } else { throw new IOException("The first parameter for knn should be a matrix."); } if (values[1] instanceof List) { List<Number> nums = (List<Number>) values[1]; vec = new double[nums.size()]; for (int i = 0; i < nums.size(); i++) { vec[i] = nums.get(i).doubleValue(); } } else { throw new IOException("The second parameter for knn should be a numeric array."); } if (values[2] instanceof Number) { k = ((Number) values[2]).intValue(); } else { throw new IOException("The third parameter for knn should be k."); } double[][] data = matrix.getData(); TreeSet<Neighbor> neighbors = new TreeSet(); for (int i = 0; i < data.length; i++) { double distance = distanceMeasure.compute(vec, data[i]); neighbors.add(new Neighbor(i, distance)); if (neighbors.size() > k) { neighbors.pollLast(); } } double[][] out = new double[neighbors.size()][]; List<String> rowLabels = matrix.getRowLabels(); List<String> newRowLabels = new ArrayList(); List<Number> distances = new ArrayList(); int i = -1; while (neighbors.size() > 0) { Neighbor neighbor = neighbors.pollFirst(); int rowIndex = neighbor.getRow(); if (rowLabels != null) { newRowLabels.add(rowLabels.get(rowIndex)); } out[++i] = data[rowIndex]; distances.add(neighbor.getDistance()); } Matrix knn = new Matrix(out); if (rowLabels != null) { knn.setRowLabels(newRowLabels); } knn.setColumnLabels(matrix.getColumnLabels()); knn.setAttribute("distances", distances); return knn; }
From source file:org.cloudata.core.tabletserver.DiskSSTable.java
public void splitMapFile(final Row.Key midRowKey, final Tablet[] createdTablets, final String columnName, List<TabletMapFile> columnMapFiles, String fileId) throws IOException { MapFileReader[] readers = new MapFileReader[columnMapFiles.size()]; MapFileWriter[] writers = new MapFileWriter[2]; TabletMapFile[] mapFiles = new TabletMapFile[2]; long targetMapFileSize = 0; try {/* w w w . j a v a 2 s.c o m*/ for (int i = 0; i < 2; i++) { GPath tempMapFilePath = new GPath( Tablet.getTabletSplitTempPath(conf, createdTablets[i].getTabletInfo()), columnName + "/" + fileId + "/"); mapFiles[i] = new TabletMapFile(conf, CloudataFileSystem.get(conf), createdTablets[i].getTabletInfo(), columnName, fileId, tempMapFilePath, numOfVersion); createdTablets[i].getDiskSSTable().addTabletMapFile(columnName, mapFiles[i]); writers[i] = mapFiles[i].getMapFileWriter(); } TreeSet<ColumnValueEntry> workPlace = new TreeSet<ColumnValueEntry>(); ColumnValue columnValue = null; for (int i = 0; i < readers.length; i++) { TabletMapFile mapFile = columnMapFiles.get(i); targetMapFileSize += mapFile.getDataFileSize(); readers[i] = mapFile.getMapFileReader(Row.Key.MIN_KEY, Row.Key.MAX_KEY); columnValue = readers[i].next(); if (columnValue != null) { workPlace.add(new ColumnValueEntry(columnValue, i)); } else { readers[i].close(); readers[i] = null; } } while (true) { if (workPlace.isEmpty()) { break; } ColumnValueEntry winnerValue = workPlace.pollFirst(); int winnerIndex = winnerValue.index; columnValue = winnerValue.columnValue; Row.Key rowKey = columnValue.getRowKey(); int nw = 0; if (rowKey.compareTo(midRowKey) > 0) { nw = writers[1].write(columnValue); } else { nw = writers[0].write(columnValue); } if (readers[winnerIndex] != null) { columnValue = readers[winnerIndex].next(); if (columnValue != null) { workPlace.add(new ColumnValueEntry(columnValue, winnerIndex)); } else { readers[winnerIndex].close(); readers[winnerIndex] = null; } } } //end of while } finally { for (int i = 0; i < writers.length; i++) { try { if (writers[i] != null) { writers[i].close(); } } catch (IOException e) { LOG.warn("IOException in closing map file writer, exception : " + e); for (int j = 0; j < mapFiles.length; j++) { mapFiles[j].delete(); } throw e; } } LOG.debug(columnName + " file splited from orig: " + targetMapFileSize + " to file1: " + mapFiles[0].getDataFileSize() + " & file2: " + mapFiles[1].getDataFileSize()); for (int i = 0; i < readers.length; i++) { try { if (readers[i] != null) { readers[i].close(); } } catch (IOException e) { throw e; } } } }
From source file:org.wso2.andes.kernel.slot.SlotManager.java
/** * Create a new slot from slotIDMap/*from www . j a v a2 s. co m*/ * * @param queueName name of the queue * @return slot object */ private Slot getFreshSlot(String queueName) { Slot slotToBeAssigned = null; TreeSetLongWrapper wrapper = slotIDMap.get(queueName); TreeSet<Long> messageIDSet; if (wrapper != null) { messageIDSet = wrapper.getLongTreeSet(); if (messageIDSet != null && !messageIDSet.isEmpty()) { //create a new slot slotToBeAssigned = new Slot(); //start msgID will be last assigned ID + 1 so that slots are created with no // message ID gaps in-between Long lastAssignedId = queueToLastAssignedIDMap.get(queueName); if (lastAssignedId != null) { slotToBeAssigned.setStartMessageId(lastAssignedId + 1); } else { slotToBeAssigned.setStartMessageId(0L); } //end messageID will be the lowest in published message ID list. Get and remove slotToBeAssigned.setEndMessageId(messageIDSet.pollFirst()); //set storage queue name (db queue to read messages from) slotToBeAssigned.setStorageQueueName(queueName); //set modified published ID map to hazelcast wrapper.setLongTreeSet(messageIDSet); slotIDMap.set(queueName, wrapper); //modify last assigned ID by queue to hazelcast queueToLastAssignedIDMap.set(queueName, slotToBeAssigned.getEndMessageId()); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from fresh pool. Slot= " + slotToBeAssigned); } } } return slotToBeAssigned; }
From source file:org.wso2.andes.kernel.slot.SlotManager.java
/** * Get an unassigned slot (slots dropped by sudden subscription closes) * * @param queueName name of the queue slot is required * @return slot or null if cannot find// ww w. j av a2 s.c o m */ private Slot getUnassignedSlot(String queueName) { Slot slotToBeAssigned = null; String lockKey = queueName + SlotManager.class; synchronized (lockKey.intern()) { TreeSetSlotWrapper unAssignedSlotWrapper = unAssignedSlotMap.get(queueName); if (unAssignedSlotWrapper != null) { TreeSet<Slot> slotsFromUnassignedSlotMap = unAssignedSlotWrapper.getSlotTreeSet(); if (slotsFromUnassignedSlotMap != null && !slotsFromUnassignedSlotMap.isEmpty()) { //Get and remove slot and update hazelcast map slotToBeAssigned = slotsFromUnassignedSlotMap.pollFirst(); unAssignedSlotWrapper.setSlotTreeSet(slotsFromUnassignedSlotMap); unAssignedSlotMap.set(queueName, unAssignedSlotWrapper); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from unAssignedSlotMap. Slot= " + slotToBeAssigned); } } } } return slotToBeAssigned; }
From source file:org.wso2.andes.kernel.slot.SlotManager.java
/** * Get an overlapped slot by nodeId and the queue name. These are slots * which are overlapped with some slots that were acquired by given node * * @param nodeId id of the node// ww w . j av a 2 s .c o m * @param queueName name of the queue slot is required * @return slot or null if not found */ private Slot getOverlappedSlot(String nodeId, String queueName) { Slot slotToBeAssigned = null; TreeSet<Slot> currentSlotList; HashMap<String, TreeSet<Slot>> queueToSlotMap; HashmapStringTreeSetWrapper wrapper = overLappedSlotMap.get(nodeId); String lockKey = nodeId + SlotManager.class; synchronized (lockKey.intern()) { if (null != wrapper) { queueToSlotMap = wrapper.getStringListHashMap(); currentSlotList = queueToSlotMap.get(queueName); if (null != currentSlotList && !currentSlotList.isEmpty()) { //get and remove slot slotToBeAssigned = currentSlotList.pollFirst(); queueToSlotMap.put(queueName, currentSlotList); //update hazelcast map wrapper.setStringListHashMap(queueToSlotMap); overLappedSlotMap.set(nodeId, wrapper); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from overlapped slot pool. Slot= " + slotToBeAssigned); } } } } return slotToBeAssigned; }
From source file:org.wso2.andes.kernel.slot.SlotManagerClusterMode.java
/** * Create a new slot from store// w ww .j av a2 s . c o m * * @param queueName name of the queue * @param nodeId id of the node * @return slot object */ private Slot getFreshSlot(String queueName, String nodeId) throws AndesException { Slot slotToBeAssigned = null; Long endMessageId = null; TreeSet<Long> messageIDSet; // Get message id set from database messageIDSet = slotAgent.getSlotBasedMessageIds(queueName); //start msgID will be last assigned ID + 1 so that slots are created with no // message ID gaps in-between long lastAssignedId = slotAgent.getQueueToLastAssignedId(queueName); /** * End message id that needs to be allocated to this slot * End messageID will be the lowest in published message ID list. Get and remove */ if (null != messageIDSet) { endMessageId = messageIDSet.pollFirst(); } /** * Check the current slot allocation not interfere into the range where expiry deletion happens. * The check is done based on the queue name and the end message id for this slot */ if (isSafeToDeliverSlots(queueName, endMessageId)) { if (null != endMessageId) { slotToBeAssigned = new Slot(); if (0L != lastAssignedId) { slotToBeAssigned.setStartMessageId(lastAssignedId + 1); } else { slotToBeAssigned.setStartMessageId(0L); } slotToBeAssigned.setEndMessageId(endMessageId); //remove polled message id from database slotAgent.deleteMessageId(queueName, slotToBeAssigned.getEndMessageId()); //set storage queue name (db queue to read messages from) slotToBeAssigned.setStorageQueueName(queueName); //modify last assigned ID by queue to database slotAgent.createSlot(slotToBeAssigned.getStartMessageId(), slotToBeAssigned.getEndMessageId(), slotToBeAssigned.getStorageQueueName(), nodeId); slotAgent.setQueueToLastAssignedId(queueName, slotToBeAssigned.getEndMessageId()); if (log.isDebugEnabled()) { log.debug("Giving a slot from fresh pool. Slot: " + slotToBeAssigned.getId()); } } } else { log.warn("Slot delivery worker is requesting the messages which are currently in deletion range for " + "queue" + queueName); } return slotToBeAssigned; }
From source file:org.wso2.andes.kernel.slot.SlotManagerStandalone.java
/** * Get an unassigned slot (slots dropped by sudden subscription closes) * * @param queueName name of the queue slot is required * @return slot or null if cannot find// ww w .j a v a 2 s .co m */ private Slot getUnassignedSlot(String queueName) { Slot slotToBeAssigned = null; TreeSet<Slot> unAssignedSlotSet = unAssignedSlotMap.get(queueName); if (null != unAssignedSlotSet) { slotToBeAssigned = unAssignedSlotSet.pollFirst(); } return slotToBeAssigned; }
From source file:org.wso2.andes.kernel.slot.SlotManagerStandalone.java
/** * Get a new slot from slotIDMap./*from w w w. j a v a 2 s .c o m*/ * * @param queueName name of the queue * @return slot object */ private Slot getFreshSlot(String queueName) { Slot slotToBeAssigned = null; Long endMessageId = null; TreeSet<Long> messageIDSet = slotIDMap.get(queueName); //start msgID will be last assigned ID + 1 so that slots are created with no // message ID gaps in-between Long lastAssignedId = queueToLastAssignedIDMap.get(queueName); // Last message id that needs to be allocated to this slot // End messageID will be the lowest in published message ID list. Get and remove if (null != messageIDSet) { endMessageId = messageIDSet.pollFirst(); } // Check the current slot allocation not interfere into the range where expiry deletion happens. // The check is done based on the queue name and the end message id for this slot. if (isSafeToDeliverSlots(queueName, endMessageId)) { if (null != endMessageId) { slotToBeAssigned = new Slot(); if (null != lastAssignedId) { slotToBeAssigned.setStartMessageId(lastAssignedId + 1); } else { slotToBeAssigned.setStartMessageId(0L); } slotToBeAssigned.setEndMessageId(endMessageId); slotToBeAssigned.setStorageQueueName(queueName); slotIDMap.put(queueName, messageIDSet); if (log.isDebugEnabled()) { log.debug(slotToBeAssigned.getEndMessageId() + " removed to slotIdMap. Current values in map " + messageIDSet); } queueToLastAssignedIDMap.put(queueName, slotToBeAssigned.getEndMessageId()); } } else { log.warn("Slot delivery worker is requesting the messages which are currently in deletion range for " + "queue " + queueName); } return slotToBeAssigned; }
From source file:org.wso2.andes.server.slot.SlotManager.java
/** * Get a slot by giving the queue name. This method first lookup the free slot pool for slots * and if there are no slots in the free slot pool then return a newly created slot * * @param queueName name of the queue/*from w w w . ja v a 2 s. c o m*/ * @return Slot object */ public Slot getSlot(String queueName, String nodeId) { Slot slotToBeAssigned; Gson gson = new GsonBuilder().create(); /** *First look in the unassigned slots pool for free slots. These slots are previously own by * other nodes */ String lockKey = queueName + SlotManager.class; synchronized (lockKey.intern()) { TreeSetStringWrapper treeSetStringWrapper = unAssignedSlotMap.get(queueName); if (treeSetStringWrapper != null) { TreeSet<String> slotsFromUnassignedSlotMap = treeSetStringWrapper.getStringTreeSet(); if (slotsFromUnassignedSlotMap != null && !slotsFromUnassignedSlotMap.isEmpty()) { slotToBeAssigned = gson.fromJson(slotsFromUnassignedSlotMap.pollFirst(), (Type) Slot.class); //update hazelcast map treeSetStringWrapper.setStringTreeSet(slotsFromUnassignedSlotMap); unAssignedSlotMap.set(queueName, treeSetStringWrapper); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from unAssignedSlotMap. Slot= " + slotToBeAssigned); } } else { slotToBeAssigned = getFreshSlot(queueName); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from fresh pool. Slot= " + slotToBeAssigned); } } } else { slotToBeAssigned = getFreshSlot(queueName); if (log.isDebugEnabled()) { log.debug("Slot Manager - giving a slot from fresh pool. Slot= " + slotToBeAssigned); } } if (null != slotToBeAssigned) { updateSlotAssignmentMap(queueName, slotToBeAssigned, nodeId); } else { if (log.isDebugEnabled()) { log.debug("Slot Manager - returns empty slot for the queue: " + queueName); } } return slotToBeAssigned; } }
From source file:org.wso2.andes.server.slot.SlotManager.java
/** * Get a new slot from slotIDMap// w w w .ja v a 2s . c om * * @param queueName name of the queue * @return slot object */ private Slot getFreshSlot(String queueName) { Slot slotToBeAssigned = null; TreeSetLongWrapper wrapper = slotIDMap.get(queueName); TreeSet<Long> messageIDSet; if (wrapper != null) { messageIDSet = wrapper.getLongTreeSet(); if (messageIDSet != null && !messageIDSet.isEmpty()) { slotToBeAssigned = new Slot(); Long lastAssignedId = queueToLastAssignedIDMap.get(queueName); if (lastAssignedId != null) { slotToBeAssigned.setStartMessageId(lastAssignedId + 1); } else { slotToBeAssigned.setStartMessageId(0L); } slotToBeAssigned.setEndMessageId(messageIDSet.pollFirst()); slotToBeAssigned.setStorageQueueName(queueName); wrapper.setLongTreeSet(messageIDSet); slotIDMap.set(queueName, wrapper); if (log.isDebugEnabled()) { log.debug(slotToBeAssigned.getEndMessageId() + " removed to slotIdMap. Current " + "values in " + "map " + messageIDSet); } queueToLastAssignedIDMap.set(queueName, slotToBeAssigned.getEndMessageId()); } } return slotToBeAssigned; }