List of usage examples for java.util BitSet set
public void set(int bitIndex)
From source file:com.asakusafw.runtime.stage.input.DefaultSplitCombiner.java
private Environment createEnvironment(Configuration configuration, List<Source> sources) throws IOException, InterruptedException { assert sources != null; Map<String, Integer> locationIds = new HashMap<>(); List<SplitDef> results = new ArrayList<>(sources.size()); for (Source source : sources) { String[] locationArray = source.getSplit().getLocations(); long length = source.getSplit().getLength(); BitSet locations = new BitSet(); if (locationArray != null) { for (String location : locationArray) { Integer id = locationIds.get(location); if (id == null) { id = locationIds.size(); locationIds.put(location, id); }//from w w w .ja v a2 s. co m locations.set(id); } } double localScore = length; double globalScore = length * configuration.nonLocalPenaltyRatio; results.add(new SplitDef(source, locations, localScore, globalScore)); } if (locationIds.isEmpty()) { locationIds.put("DUMMY-LOCATION", locationIds.size()); //$NON-NLS-1$ } String[] locations = new String[locationIds.size()]; for (Map.Entry<String, Integer> entry : locationIds.entrySet()) { locations[entry.getValue()] = entry.getKey(); } SplitDef[] splitDefs = results.toArray(new SplitDef[results.size()]); SlotDef[] slotDefs = resolveSlots(configuration.slotsPerInput, locations, results); return new Environment(locations, splitDefs, slotDefs, configuration); }
From source file:dr.app.tools.AntigenicPlotter.java
public AntigenicPlotter(int burnin, boolean tabFormat, boolean discreteModel, final String inputFileName, final String treeFileName, final String outputFileName) throws IOException { double[][] reference = null; List<String> tipLabels = null; if (treeFileName != null) { System.out.println("Reading tree file..."); NexusImporter importer = new NexusImporter(new FileReader(treeFileName)); try {//from w w w .j a v a2s . c o m Tree tree = importer.importNextTree(); reference = new double[tree.getExternalNodeCount()][2]; tipLabels = new ArrayList<String>(); for (int i = 0; i < tree.getExternalNodeCount(); i++) { NodeRef tip = tree.getExternalNode(i); tipLabels.add(tree.getNodeTaxon(tip).getId()); reference[i][0] = (Double) tree.getNodeAttribute(tip, "antigenic1"); reference[i][1] = (Double) tree.getNodeAttribute(tip, "antigenic2"); } } catch (Importer.ImportException e) { e.printStackTrace(); return; } } System.out.println("Reading log file..."); FileReader fileReader = new FileReader(inputFileName); try { File file = new File(inputFileName); LogFileTraces traces = new LogFileTraces(inputFileName, file); traces.loadTraces(); if (burnin == -1) { burnin = (int) (traces.getMaxState() / 10); } traces.setBurnIn(burnin); System.out.println(); System.out.println("burnIn <= " + burnin); System.out.println("maxState = " + traces.getMaxState()); System.out.println(); int traceCount = traces.getTraceCount(); if (discreteModel) { // for the discrete model, there are 4 sets of traces, pairs coordinates, cluster allocations, and cluster sizes traceCount /= 4; } else { // for continuous, just pairs of coordinates traceCount /= 2; } int stateCount = traces.getStateCount(); double[][][] data; String[] labels = new String[traceCount]; if (tipLabels != null) { data = new double[stateCount][tipLabels.size()][2]; } else { data = new double[stateCount][traceCount][2]; } for (int i = 0; i < traceCount; i++) { String name = traces.getTraceName(i * 2); name = name.substring(0, name.length() - 1); if (tipLabels != null) { int index = tipLabels.indexOf(name); if (index != -1) { for (int j = 0; j < stateCount; j++) { data[j][index][0] = traces.getStateValue(i * 2, j); data[j][index][1] = traces.getStateValue((i * 2) + 1, j); } } } else { for (int j = 0; j < stateCount; j++) { data[j][i][0] = traces.getStateValue(i * 2, j); data[j][i][1] = traces.getStateValue((i * 2) + 1, j); } labels[i] = name; } } int[][] clusterIndices = null; int[][] clusterSizes = null; if (discreteModel) { clusterIndices = new int[stateCount][traceCount]; clusterSizes = new int[stateCount][traceCount]; for (int i = 0; i < traceCount; i++) { for (int j = 0; j < stateCount; j++) { clusterIndices[j][i] = (int) traces.getStateValue((traceCount * 2) + i, j); clusterSizes[j][i] = (int) traces.getStateValue((traceCount * 3) + i, j); } } Map<BitSet, Integer> clusterMap = new HashMap<BitSet, Integer>(); for (int i = 0; i < stateCount; i++) { BitSet[] clusters = new BitSet[clusterIndices[i].length]; for (int j = 0; j < clusterIndices[i].length; j++) { BitSet bits = clusters[clusterIndices[i][j]]; if (bits == null) { bits = new BitSet(); clusters[clusterIndices[i][j]] = bits; } bits.set(j); Integer count = clusterMap.get(bits); if (count == null) { count = 0; } clusterMap.put(bits, count + 1); } Arrays.sort(clusters, new Comparator<BitSet>() { public int compare(BitSet bitSet1, BitSet bitSet2) { if (bitSet1 == null) { return -1; } if (bitSet2 == null) { return 1; } return bitSet2.cardinality() - bitSet1.cardinality(); } }); } for (BitSet bits : clusterMap.keySet()) { int count = clusterMap.get(bits); if (count > 1) { System.out.print(count); for (int i = bits.nextSetBit(0); i >= 0; i = bits.nextSetBit(i + 1)) { System.out.print("\t" + labels[i]); } System.out.println(); } } } if (tipLabels != null) { labels = new String[tipLabels.size()]; tipLabels.toArray(labels); } if (reference != null) { procrustinate(data, reference); } else { procrustinate(data); } if (tabFormat) { writeTabformat(outputFileName, labels, data); } else { if (discreteModel) { writeKML(outputFileName, labels, data, clusterIndices, clusterSizes); } else { writeKML(outputFileName, labels, data); } } } catch (Exception e) { System.err.println("Error Parsing Input File: " + e.getMessage()); e.printStackTrace(System.err); return; } fileReader.close(); }
From source file:io.viewserver.operators.group.GroupByOperator.java
@Override protected void processConfig(IGroupByConfig config) { if (configChanged(config)) { List<String> groupBy = config.getGroupBy(); if (groupIdComponents == null || groupIdComponents.length < groupBy.size()) { groupIdComponents = new int[groupBy.size()]; }/*from w w w .jav a2s .co m*/ this.groupBy = groupBy.toArray(new String[groupBy.size()]); input.groupByColumns = new ColumnHolder[this.groupBy.length]; List<IGroupByConfig.Summary> summaries = config.getSummaries(); this.summaryConfig = summaries.toArray(new IGroupByConfig.Summary[summaries.size()]); this.countColumnName = config.getCountColumnName(); subtotals.clear(); if (config.getSubtotals() != null && config.getSubtotals().size() != 0) { for (String subtotal : config.getSubtotals()) { BitSet subtotalColumns = new BitSet(); if (subtotal != null && !"".equals(subtotal)) { for (String subtotalColumn : subtotal.split("\\|")) { int index = groupBy.indexOf(subtotalColumn); if (index == -1) { throw new IllegalArgumentException( "Invalid subtotal configuration - no group column '" + subtotalColumn + "'"); } subtotalColumns.set(index); } } subtotals.add(subtotalColumns); } } else { BitSet subtotalColumns = new BitSet(); for (int i = 0; i < groupBy.size(); i++) { subtotalColumns.set(i); } subtotals.add(subtotalColumns); } input.resetSchema(); } }
From source file:itemsetmining.itemset.ItemsetTree.java
/** * Pearson's chi-squared test for itemset independence. This tests the * empirical itemset distribution against the independence model. * * <p>/* w w w . j ava 2s .co m*/ * N.B. the chi-squared distribution has one degree of freedom. * * @see S. Brin et al. Beyond Market Baskets: Generalizing Association Rules * to Correlations */ private double recursiveChiSquared(final int n, final BitSet cell, final int[] sortedItems, final Multiset<Integer> singletons) { double chiSquared = 0.; if (n == sortedItems.length) { double pInd = noTransactions; final int[] inItems = new int[cell.cardinality()]; final int[] outItems = new int[n - cell.cardinality()]; int i = 0, j = 0; for (int k = 0; k < n; k++) { if (cell.get(k)) { inItems[i] = sortedItems[k]; i++; pInd *= singletons.count(sortedItems[k]) / (double) noTransactions; } else { outItems[j] = sortedItems[k]; j++; pInd *= (noTransactions - singletons.count(sortedItems[k])) / (double) noTransactions; } } final double pEmp = countEmpirical(inItems, outItems, root, new int[0]); chiSquared = ((pEmp - pInd) * (pEmp - pInd)) / pInd; } else { final BitSet celln = (BitSet) cell.clone(); celln.set(n); chiSquared += recursiveChiSquared(n + 1, celln, sortedItems, singletons); chiSquared += recursiveChiSquared(n + 1, cell, sortedItems, singletons); } return chiSquared; }
From source file:org.wso2.andes.subscription.TopicSubscriptionBitMapStore.java
/** * This methods adds a constituent table with only null and other constituents. * This is required when a message comes with more than the available number of constituents. If wildcard * subscriptions are available for those, they should match. Hence need to create these empty constituent tables. *//*from ww w . j a v a 2 s . c om*/ private void addEmptyConstituentTable() { int noOfSubscriptions = subscriptionList.size(); Map<String, BitSet> constituentTable = new HashMap<String, BitSet>(); BitSet nullBitSet = new BitSet(noOfSubscriptions); BitSet otherBitSet = new BitSet(noOfSubscriptions); if (noOfSubscriptions > 0) { // Null constituent will always be true for empty constituents, hence need to flip nullBitSet.flip(0, noOfSubscriptions - 1); for (int subscriptionIndex = 0; subscriptionIndex < noOfSubscriptions; subscriptionIndex++) { // For 'other', if subscribers last constituent is multi level wild card then matching String[] allConstituent = subscriptionConstituents.get(subscriptionIndex); String lastConstituent = allConstituent[allConstituent.length - 1]; if (multiLevelWildCard.equals(lastConstituent)) { otherBitSet.set(subscriptionIndex); } else { otherBitSet.set(subscriptionIndex, false); } } } constituentTable.put(NULL_CONSTITUENT, nullBitSet); constituentTable.put(OTHER_CONSTITUENT, otherBitSet); constituentTables.add(constituentTable); }
From source file:model.DecomposableModel.java
public double messageLengthDiffIfAdding(Integer a, Integer b, MessageLengthFactorialComputer computer, boolean verbose) { BitSet Sab = graph.getSeparator(a, b); BitSet Sabua = (BitSet) Sab.clone(); BitSet Sabub = (BitSet) Sab.clone(); BitSet Sabuaub = (BitSet) Sab.clone(); Sabua.set(a); Sabub.set(b);//from w w w . jav a 2 s . c o m Sabuaub.set(a); Sabuaub.set(b); double lengthFrequencies, lengthPositionData; // model // System.out.println("graph"); //all graphs equally likely lengthFrequencies = this.nbParametersDiffIfAdding(a, b) * computer.getLogFromTable(computer.getNbInstances() + 1); lengthPositionData = 0.0; lengthPositionData += computer.computeLengthData(Sab); lengthPositionData -= computer.computeLengthData(Sabua); lengthPositionData -= computer.computeLengthData(Sabub); lengthPositionData += computer.computeLengthData(Sabuaub); assert lengthPositionData >= 0; if (verbose) { System.out.println("adding (" + a + "," + b + ")"); System.out.println("#param diff = " + this.nbParametersDiffIfAdding(a, b)); System.out.println("diff length frequencies=" + lengthFrequencies); System.out.println("diff length data|freq=" + lengthPositionData); } encodingLength = lengthFrequencies + lengthPositionData; return encodingLength; }
From source file:org.wso2.andes.subscription.ClusterSubscriptionBitMapHandler.java
/** * This methods adds a constituent table with only null and other constituents. * This is required when a message comes with more than the available number of constituents. If wildcard * subscriptions are available for those, they should match. Hence need to create these empty constituent tables. *///from w w w . j a v a 2s. c o m private void addEmptyConstituentTable() { int noOfSubscriptions = wildCardSubscriptionList.size(); Map<String, BitSet> constituentTable = new HashMap<String, BitSet>(); BitSet nullBitSet = new BitSet(noOfSubscriptions); BitSet otherBitSet = new BitSet(noOfSubscriptions); if (noOfSubscriptions > 0) { // Null constituent will always be true for empty constituents, hence need to flip nullBitSet.flip(0, noOfSubscriptions - 1); for (int subscriptionIndex = 0; subscriptionIndex < noOfSubscriptions; subscriptionIndex++) { // For 'other', if subscribers last constituent is multi level wild card then matching String[] allConstituent = subscriptionConstituents.get(subscriptionIndex); String lastConstituent = allConstituent[allConstituent.length - 1]; if (multiLevelWildCard.equals(lastConstituent)) { otherBitSet.set(subscriptionIndex); } else { otherBitSet.set(subscriptionIndex, false); } } } constituentTable.put(NULL_CONSTITUENT, nullBitSet); constituentTable.put(OTHER_CONSTITUENT, otherBitSet); constituentTables.add(constituentTable); }
From source file:model.DecomposableModel.java
public long nbParametersDiffIfAdding(Integer a, Integer b, BitSet Sab) { // System.out.println("computing actual entropy"); BitSet Sabua = (BitSet) Sab.clone(); BitSet Sabub = (BitSet) Sab.clone(); BitSet Sabuaub = (BitSet) Sab.clone(); Sabua.set(a); Sabub.set(b);//from w w w .j av a2s. c o m Sabuaub.set(a); Sabuaub.set(b); long diffNbParameters = 0; // Sab int tmpNBDF = 1; for (int var = Sab.nextSetBit(0); var >= 0; var = Sab.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters += tmpNBDF; // Sab + a tmpNBDF = 1; for (int var = Sabua.nextSetBit(0); var >= 0; var = Sabua.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters -= tmpNBDF; // Sab + a + b tmpNBDF = 1; for (int var = Sabuaub.nextSetBit(0); var >= 0; var = Sabuaub.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters += tmpNBDF; // Sab + b tmpNBDF = 1; for (int var = Sabub.nextSetBit(0); var >= 0; var = Sabub.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters -= tmpNBDF; return diffNbParameters; }
From source file:model.DecomposableModel.java
public long nbParametersDiffIfAdding(Integer a, Integer b) { // System.out.println("computing actual entropy"); BitSet Sab = graph.getSeparator(a, b); BitSet Sabua = (BitSet) Sab.clone(); BitSet Sabub = (BitSet) Sab.clone(); BitSet Sabuaub = (BitSet) Sab.clone(); Sabua.set(a); Sabub.set(b);/* w w w. ja va 2 s .co m*/ Sabuaub.set(a); Sabuaub.set(b); long diffNbParameters = 0; // Sab int tmpNBDF = 1; for (int var = Sab.nextSetBit(0); var >= 0; var = Sab.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters += tmpNBDF; // Sab + a tmpNBDF = 1; for (int var = Sabua.nextSetBit(0); var >= 0; var = Sabua.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters -= tmpNBDF; // Sab + a + b tmpNBDF = 1; for (int var = Sabuaub.nextSetBit(0); var >= 0; var = Sabuaub.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters += tmpNBDF; // Sab + b tmpNBDF = 1; for (int var = Sabub.nextSetBit(0); var >= 0; var = Sabub.nextSetBit(var + 1)) { tmpNBDF *= dimensionsForVariables[var]; } tmpNBDF = tmpNBDF - 1; diffNbParameters -= tmpNBDF; return diffNbParameters; }
From source file:org.apache.tez.runtime.library.common.writers.TestUnorderedPartitionedKVWriter.java
public void textTest(int numRegularRecords, int numPartitions, long availableMemory, int numLargeKeys, int numLargevalues, int numLargeKvPairs) throws IOException, InterruptedException { Partitioner partitioner = new HashPartitioner(); ApplicationId appId = ApplicationId.newInstance(10000, 1); TezCounters counters = new TezCounters(); String uniqueId = UUID.randomUUID().toString(); OutputContext outputContext = createMockOutputContext(counters, appId, uniqueId); Random random = new Random(); Configuration conf = createConfiguration(outputContext, Text.class, Text.class, shouldCompress, -1, HashPartitioner.class); CompressionCodec codec = null;//from ww w . j a va 2 s . c o m if (shouldCompress) { codec = new DefaultCodec(); ((Configurable) codec).setConf(conf); } int numRecordsWritten = 0; Map<Integer, Multimap<String, String>> expectedValues = new HashMap<Integer, Multimap<String, String>>(); for (int i = 0; i < numPartitions; i++) { expectedValues.put(i, LinkedListMultimap.<String, String>create()); } UnorderedPartitionedKVWriter kvWriter = new UnorderedPartitionedKVWriterForTest(outputContext, conf, numPartitions, availableMemory); int sizePerBuffer = kvWriter.sizePerBuffer; BitSet partitionsWithData = new BitSet(numPartitions); Text keyText = new Text(); Text valText = new Text(); for (int i = 0; i < numRegularRecords; i++) { String key = createRandomString(Math.abs(random.nextInt(10))); String val = createRandomString(Math.abs(random.nextInt(20))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write Large key records for (int i = 0; i < numLargeKeys; i++) { String key = createRandomString(sizePerBuffer + Math.abs(random.nextInt(100))); String val = createRandomString(Math.abs(random.nextInt(20))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write Large val records for (int i = 0; i < numLargevalues; i++) { String key = createRandomString(Math.abs(random.nextInt(10))); String val = createRandomString(sizePerBuffer + Math.abs(random.nextInt(100))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } // Write records where key + val are large (but both can fit in the buffer individually) for (int i = 0; i < numLargeKvPairs; i++) { String key = createRandomString(sizePerBuffer / 2 + Math.abs(random.nextInt(100))); String val = createRandomString(sizePerBuffer / 2 + Math.abs(random.nextInt(100))); keyText.set(key); valText.set(val); int partition = partitioner.getPartition(keyText, valText, numPartitions); partitionsWithData.set(partition); expectedValues.get(partition).put(key, val); kvWriter.write(keyText, valText); numRecordsWritten++; } List<Event> events = kvWriter.close(); verify(outputContext, never()).fatalError(any(Throwable.class), any(String.class)); TezCounter outputLargeRecordsCounter = counters.findCounter(TaskCounter.OUTPUT_LARGE_RECORDS); assertEquals(numLargeKeys + numLargevalues + numLargeKvPairs, outputLargeRecordsCounter.getValue()); // Validate the event assertEquals(1, events.size()); assertTrue(events.get(0) instanceof CompositeDataMovementEvent); CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) events.get(0); assertEquals(0, cdme.getSourceIndexStart()); assertEquals(numPartitions, cdme.getCount()); DataMovementEventPayloadProto eventProto = DataMovementEventPayloadProto .parseFrom(ByteString.copyFrom(cdme.getUserPayload())); assertFalse(eventProto.hasData()); BitSet emptyPartitionBits = null; if (partitionsWithData.cardinality() != numPartitions) { assertTrue(eventProto.hasEmptyPartitions()); byte[] emptyPartitions = TezCommonUtils .decompressByteStringToByteArray(eventProto.getEmptyPartitions()); emptyPartitionBits = TezUtilsInternal.fromByteArray(emptyPartitions); assertEquals(numPartitions - partitionsWithData.cardinality(), emptyPartitionBits.cardinality()); } else { assertFalse(eventProto.hasEmptyPartitions()); emptyPartitionBits = new BitSet(numPartitions); } assertEquals(HOST_STRING, eventProto.getHost()); assertEquals(SHUFFLE_PORT, eventProto.getPort()); assertEquals(uniqueId, eventProto.getPathComponent()); // Verify the data // Verify the actual data TezTaskOutput taskOutput = new TezTaskOutputFiles(conf, uniqueId); Path outputFilePath = null; Path spillFilePath = null; try { outputFilePath = taskOutput.getOutputFile(); } catch (DiskErrorException e) { if (numRecordsWritten > 0) { fail(); } else { // Record checking not required. return; } } try { spillFilePath = taskOutput.getOutputIndexFile(); } catch (DiskErrorException e) { if (numRecordsWritten > 0) { fail(); } else { // Record checking not required. return; } } // Special case for 0 records. TezSpillRecord spillRecord = new TezSpillRecord(spillFilePath, conf); DataInputBuffer keyBuffer = new DataInputBuffer(); DataInputBuffer valBuffer = new DataInputBuffer(); Text keyDeser = new Text(); Text valDeser = new Text(); for (int i = 0; i < numPartitions; i++) { if (emptyPartitionBits.get(i)) { continue; } TezIndexRecord indexRecord = spillRecord.getIndex(i); FSDataInputStream inStream = FileSystem.getLocal(conf).open(outputFilePath); inStream.seek(indexRecord.getStartOffset()); IFile.Reader reader = new IFile.Reader(inStream, indexRecord.getPartLength(), codec, null, null, false, 0, -1); while (reader.nextRawKey(keyBuffer)) { reader.nextRawValue(valBuffer); keyDeser.readFields(keyBuffer); valDeser.readFields(valBuffer); int partition = partitioner.getPartition(keyDeser, valDeser, numPartitions); assertTrue(expectedValues.get(partition).remove(keyDeser.toString(), valDeser.toString())); } inStream.close(); } for (int i = 0; i < numPartitions; i++) { assertEquals(0, expectedValues.get(i).size()); expectedValues.remove(i); } assertEquals(0, expectedValues.size()); }