List of usage examples for java.util BitSet cardinality
public int cardinality()
From source file:org.apache.hadoop.record.TestRecordWritable.java
public void testFormat() throws Exception { JobConf job = new JobConf(conf); FileSystem fs = FileSystem.getLocal(conf); Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred"); Path file = new Path(dir, "test.seq"); int seed = new Random().nextInt(); //LOG.info("seed = "+seed); Random random = new Random(seed); fs.delete(dir, true);//w ww . j a va2 s . co m FileInputFormat.setInputPaths(job, dir); // for a variety of lengths for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) { // create a file with length entries SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, file, RecInt.class, RecBuffer.class); try { for (int i = 0; i < length; i++) { RecInt key = new RecInt(); key.setData(i); byte[] data = new byte[random.nextInt(10)]; random.nextBytes(data); RecBuffer value = new RecBuffer(); value.setData(new Buffer(data)); writer.append(key, value); } } finally { writer.close(); } // try splitting the file in a variety of sizes InputFormat<RecInt, RecBuffer> format = new SequenceFileInputFormat<RecInt, RecBuffer>(); RecInt key = new RecInt(); RecBuffer value = new RecBuffer(); for (int i = 0; i < 3; i++) { int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1; InputSplit[] splits = format.getSplits(job, numSplits); // check each split BitSet bits = new BitSet(length); for (int j = 0; j < splits.length; j++) { RecordReader<RecInt, RecBuffer> reader = format.getRecordReader(splits[j], job, Reporter.NULL); try { int count = 0; while (reader.next(key, value)) { assertFalse("Key in multiple partitions.", bits.get(key.getData())); bits.set(key.getData()); count++; } } finally { reader.close(); } } assertEquals("Some keys in no partition.", length, bits.cardinality()); } } }
From source file:dendroscope.autumn.hybridnetwork.ComputeHybridizationNetwork.java
/** * run the algorithm//from w w w. jav a 2 s .c om * * @param tree1 * @param tree2 * @param hybridizationNumber * @return reduced trees */ private TreeData[] run(TreeData tree1, TreeData tree2, int upperBound, Single<Integer> hybridizationNumber) throws IOException, CanceledException { verbose = ProgramProperties.get("verbose-HL", false); Taxa allTaxa = new Taxa(); Pair<Root, Root> roots = PreProcess.apply(tree1, tree2, allTaxa); Root root1 = roots.getFirst(); Root root2 = roots.getSecond(); if (root1.getOutDegree() == 1 && root1.getFirstOutEdge().getTarget().getOutDegree() > 0) { Root tmp = (Root) root1.getFirstOutEdge().getTarget(); root1.deleteNode(); root1 = tmp; } if (root2.getOutDegree() == 1 && root2.getFirstOutEdge().getTarget().getOutDegree() > 0) { Root tmp = (Root) root2.getFirstOutEdge().getTarget(); root2.deleteNode(); root2 = tmp; } BitSet onlyTree1 = Cluster.setminus(root1.getTaxa(), root2.getTaxa()); BitSet onlyTree2 = Cluster.setminus(root2.getTaxa(), root1.getTaxa()); if (root1.getTaxa().cardinality() == onlyTree1.cardinality()) throw new IOException("None of the taxa in second tree are contained in first tree"); if (root2.getTaxa().cardinality() == onlyTree2.cardinality()) throw new IOException("None of the taxa in first tree are contained in second tree"); if (onlyTree1.cardinality() > 0) { System.err.println("Killing all taxa only present in first tree: " + onlyTree1.cardinality()); for (int t = onlyTree1.nextSetBit(0); t != -1; t = onlyTree1.nextSetBit(t + 1)) { RemoveTaxon.apply(root1, 1, t); } } if (onlyTree2.cardinality() > 0) { System.err.println("Killing all taxa only present in second tree: " + onlyTree2.cardinality()); for (int t = onlyTree2.nextSetBit(0); t != -1; t = onlyTree2.nextSetBit(t + 1)) { RemoveTaxon.apply(root2, 2, t); } } // run the refine algorithm System.err.println("Computing common refinement of both trees"); Refine.apply(root1, root2); if (tree1.getRoot() == null || tree2.getRoot() == null) { throw new IOException( "Can't compute hybridization networks, at least one of the trees is empty or unrooted"); } // we maintain both trees in lexicographic order for ease of comparison root1.reorderSubTree(); root2.reorderSubTree(); System.err.println( "Computing hybridization networks using Autumn algorithm (Autumn algorithm, Huson and Linz, 2016)..."); progressListener.setTasks("Computing hybridization networks", "(Unknown how long this will really take)"); progressListener.setMaximum(20); progressListener.setProgress(0); long startTime = System.currentTimeMillis(); nextTime = startTime + waitTime; Set<Root> result = new TreeSet<>(new NetworkComparator()); int h = computeRec(root1, root2, false, getAllAliveTaxa(root1, root2), upperBound, result, ">"); fixOrdering(result); if (false) { Collection<Root> maafs = MAAFUtils.computeAllMAAFs(result); System.err.println("MAAFs before:"); for (Root root : maafs) { System.err.println(root.toStringNetworkFull()); } } int numberOfDuplicatesRemoved = MAAFUtils.removeDuplicateMAAFs(result, false); if (numberOfDuplicatesRemoved > 0) System.err.println("MAAF duplicates removed: " + numberOfDuplicatesRemoved); if (false) { Collection<Root> maafs = MAAFUtils.computeAllMAAFs(result); System.err.println("MAAFs after:"); for (Root root : maafs) { System.err.println(root.toStringNetworkFull()); } } fixOrdering(result); BitSet missingTaxa = Cluster.union(onlyTree1, onlyTree2); if (missingTaxa.cardinality() > 0) { System.err.println("Reattaching killed taxa: " + missingTaxa.cardinality()); for (Root r : result) { for (int t = missingTaxa.nextSetBit(0); t != -1; t = missingTaxa.nextSetBit(t + 1)) { RemoveTaxon.unapply(r, t); } } } System.err.println("Hybridization number: " + h); hybridizationNumber.set(h); System.err.println("Total networks: " + result.size()); System.err.println("Time: " + ((System.currentTimeMillis() - startTime) / 1000) + " secs"); System.err.println( "(Size lookup table: " + lookupTable.size() + ", number of times used: " + numberOfLookups + ")"); lookupTable.clear(); System.gc(); if (false) { System.err.println("Networks:"); for (Root root : result) { System.err.println(root.toStringNetworkFull()); } } System.gc(); List<TreeData> list = PostProcess.apply(result.toArray(new Root[result.size()]), allTaxa, false); return list.toArray(new TreeData[list.size()]); }
From source file:uk.ac.ebi.atlas.experimentpage.baseline.genedistribution.BarChartTrader.java
protected int countGenesAboveCutoff(Map<FactorGroup, BitSet> geneBitSets, Set<Factor> filterFactors, Set<Factor> selectedFactors) { BitSet expressedGenesBitSet = new BitSet(AVERAGE_GENES_IN_EXPERIMENT); // get the union of genes expressed for all FactorGroups that are in the slice and have been selected for (FactorGroup factorGroup : geneBitSets.keySet()) { boolean factorGroupContainsAllFilterFactors = CollectionUtils.isEmpty(filterFactors) || factorGroup.containsAll(filterFactors); boolean factorGroupOverlapsSelectedFactors = CollectionUtils.isEmpty(selectedFactors) || factorGroup.overlapsWith(selectedFactors); if (factorGroupContainsAllFilterFactors && factorGroupOverlapsSelectedFactors) { expressedGenesBitSet.or(geneBitSets.get(factorGroup)); }//from ww w . j av a2 s . co m } return expressedGenesBitSet.cardinality(); }
From source file:org.apache.tez.runtime.library.common.sort.impl.dflt.TestDefaultSorter.java
public void testEmptyCaseFileLengthsHelper(int numPartitions, String[] keys, String[] values) throws IOException { OutputContext context = createTezOutputContext(); MemoryUpdateCallbackHandler handler = new MemoryUpdateCallbackHandler(); conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 1); context.requestInitialMemory(/*from w ww . ja va2 s. com*/ ExternalSorter.getInitialMemoryRequirement(conf, context.getTotalMemoryAvailableToTask()), handler); String auxService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT); SorterWrapper sorterWrapper = new SorterWrapper(context, conf, numPartitions, handler.getMemoryAssigned()); DefaultSorter sorter = sorterWrapper.getSorter(); assertEquals("Key and Values must have the same number of elements", keys.length, values.length); BitSet keyRLEs = new BitSet(keys.length); for (int i = 0; i < keys.length; i++) { boolean isRLE = sorterWrapper.writeKeyValue(new Text(keys[i]), new Text(values[i])); keyRLEs.set(i, isRLE); } sorterWrapper.close(); List<Event> events = new ArrayList<>(); String pathComponent = (context.getUniqueIdentifier() + "_" + 0); ShuffleUtils.generateEventOnSpill(events, true, true, context, 0, sorter.indexCacheList.get(0), 0, true, pathComponent, sorter.getPartitionStats(), sorter.reportDetailedPartitionStats(), auxService, TezCommonUtils.newBestCompressionDeflater()); CompositeDataMovementEvent compositeDataMovementEvent = (CompositeDataMovementEvent) events.get(1); ByteBuffer bb = compositeDataMovementEvent.getUserPayload(); ShuffleUserPayloads.DataMovementEventPayloadProto shufflePayload = ShuffleUserPayloads.DataMovementEventPayloadProto .parseFrom(ByteString.copyFrom(bb)); if (shufflePayload.hasEmptyPartitions()) { byte[] emptyPartitionsBytesString = TezCommonUtils .decompressByteStringToByteArray(shufflePayload.getEmptyPartitions()); BitSet emptyPartitionBitSet = TezUtilsInternal.fromByteArray(emptyPartitionsBytesString); Assert.assertEquals("Number of empty partitions did not match!", emptyPartitionBitSet.cardinality(), sorterWrapper.getEmptyPartitionsCount()); } else { Assert.assertEquals(sorterWrapper.getEmptyPartitionsCount(), 0); } // Each non-empty partition adds 4 bytes for header, 2 bytes for EOF_MARKER, 4 bytes for checksum int expectedFileOutLength = sorterWrapper.getNonEmptyPartitionsCount() * 10; for (int i = 0; i < keys.length; i++) { // Each Record adds 1 byte for key length, 1 byte Text overhead (length), key.length bytes for key expectedFileOutLength += keys[i].length() + 2; // Each Record adds 1 byte for value length, 1 byte Text overhead (length), value.length bytes for value expectedFileOutLength += values[i].length() + 2; } assertEquals("Unexpected Output File Size!", localFs.getFileStatus(sorter.getFinalOutputFile()).getLen(), expectedFileOutLength); assertEquals(sorter.getNumSpills(), 1); verifyCounters(sorter, context); }
From source file:org.apache.carbondata.hadoop.api.CarbonTableInputFormat.java
/** * Read data in one segment. For alter table partition statement * @param job//w w w . j a v a 2s . c om * @param targetSegment * @param oldPartitionIdList get old partitionId before partitionInfo was changed * @return */ public List<InputSplit> getSplitsOfOneSegment(JobContext job, String targetSegment, List<Integer> oldPartitionIdList, PartitionInfo partitionInfo) { List<Segment> invalidSegments = new ArrayList<>(); List<UpdateVO> invalidTimestampsList = new ArrayList<>(); try { carbonTable = getOrCreateCarbonTable(job.getConfiguration()); ReadCommittedScope readCommittedScope = getReadCommitted(job, carbonTable.getAbsoluteTableIdentifier()); this.readCommittedScope = readCommittedScope; List<Segment> segmentList = new ArrayList<>(); Segment segment = Segment.getSegment(targetSegment, carbonTable.getTablePath()); segmentList.add(new Segment(segment.getSegmentNo(), segment.getSegmentFileName(), readCommittedScope)); setSegmentsToAccess(job.getConfiguration(), segmentList); // process and resolve the expression Expression filter = getFilterPredicates(job.getConfiguration()); CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration()); // this will be null in case of corrupt schema file. if (null == carbonTable) { throw new IOException("Missing/Corrupt schema file for table."); } carbonTable.processFilterExpression(filter, null, null); // prune partitions for filter query on partition table String partitionIds = job.getConfiguration().get(ALTER_PARTITION_ID); // matchedPartitions records partitionIndex, not partitionId BitSet matchedPartitions = null; if (partitionInfo != null) { matchedPartitions = setMatchedPartitions(partitionIds, filter, partitionInfo, oldPartitionIdList); if (matchedPartitions != null) { if (matchedPartitions.cardinality() == 0) { return new ArrayList<InputSplit>(); } else if (matchedPartitions.cardinality() == partitionInfo.getNumPartitions()) { matchedPartitions = null; } } } FilterResolverIntf filterInterface = carbonTable.resolveFilter(filter); // do block filtering and get split List<InputSplit> splits = getSplits(job, filterInterface, segmentList, matchedPartitions, partitionInfo, oldPartitionIdList, new SegmentUpdateStatusManager(carbonTable)); // pass the invalid segment to task side in order to remove index entry in task side if (invalidSegments.size() > 0) { for (InputSplit split : splits) { ((CarbonInputSplit) split).setInvalidSegments(invalidSegments); ((CarbonInputSplit) split).setInvalidTimestampRange(invalidTimestampsList); } } return splits; } catch (IOException e) { throw new RuntimeException("Can't get splits of the target segment ", e); } }
From source file:edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule.java
private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) { List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>(); previousCandidates.addAll(candidates); candidates.clear();/* w w w. java 2 s .c o m*/ boolean validCandidate = false; for (Mutable<ILogicalOperator> op : opList) { List<Mutable<ILogicalOperator>> inputs = op.getValue().getInputs(); for (int i = 0; i < inputs.size(); i++) { Mutable<ILogicalOperator> inputRef = inputs.get(i); validCandidate = false; for (Mutable<ILogicalOperator> candidate : previousCandidates) { // if current input is in candidates if (inputRef.getValue().equals(candidate.getValue())) { if (inputs.size() == 1) { validCandidate = true; } else { BitSet candidateInputBitMap = opToCandidateInputs.get(op); if (candidateInputBitMap == null) { candidateInputBitMap = new BitSet(inputs.size()); opToCandidateInputs.put(op, candidateInputBitMap); } candidateInputBitMap.set(i); if (candidateInputBitMap.cardinality() == inputs.size()) { validCandidate = true; } } break; } } } if (!validCandidate) continue; if (!candidates.contains(op)) candidates.add(op); } }
From source file:org.apache.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule.java
private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) { List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>(); previousCandidates.addAll(candidates); candidates.clear();// ww w . j a v a2 s . c o m boolean validCandidate = false; for (Mutable<ILogicalOperator> op : opList) { List<Mutable<ILogicalOperator>> inputs = op.getValue().getInputs(); for (int i = 0; i < inputs.size(); i++) { Mutable<ILogicalOperator> inputRef = inputs.get(i); validCandidate = false; for (Mutable<ILogicalOperator> candidate : previousCandidates) { // if current input is in candidates if (inputRef.getValue().equals(candidate.getValue())) { if (inputs.size() == 1) { validCandidate = true; } else { BitSet candidateInputBitMap = opToCandidateInputs.get(op); if (candidateInputBitMap == null) { candidateInputBitMap = new BitSet(inputs.size()); opToCandidateInputs.put(op, candidateInputBitMap); } candidateInputBitMap.set(i); if (candidateInputBitMap.cardinality() == inputs.size()) { validCandidate = true; } } break; } } } if (!validCandidate) { continue; } if (!candidates.contains(op)) { candidates.add(op); } } }
From source file:com.bittorrent.mpetazzoni.client.SharedTorrent.java
/** * Bit field availability handler.//w w w.j a v a 2 s . c om * * <p> * Handle updates in piece availability from a peer's BITFIELD message. * When this happens, we need to mark in all the pieces the peer has that * they can be reached through this peer, thus augmenting the global * availability of pieces. * </p> * * @param peer The peer we got the update from. * @param availablePieces The pieces availability bit field of the peer. */ @Override public synchronized void handleBitfieldAvailability(SharingPeer peer, BitSet availablePieces) { // Determine if the peer is interesting for us or not, and notify it. BitSet interesting = (BitSet) availablePieces.clone(); interesting.andNot(this.completedPieces); interesting.andNot(this.requestedPieces); if (interesting.cardinality() == 0) { peer.notInteresting(); } else { peer.interesting(); } // Record that the peer has all the pieces it told us it had. for (int i = availablePieces.nextSetBit(0); i >= 0; i = availablePieces.nextSetBit(i + 1)) { this.rarest.remove(this.pieces[i]); this.pieces[i].seenAt(peer); this.rarest.add(this.pieces[i]); } logger.trace("Peer {} contributes {} piece(s) ({} interesting) " + "[completed={}; available={}/{}].", new Object[] { peer, availablePieces.cardinality(), interesting.cardinality(), this.completedPieces.cardinality(), this.getAvailablePieces().cardinality(), this.pieces.length }); }
From source file:com.opengamma.analytics.math.minimization.UncoupledParameterTransforms.java
/** * * @param startValues fixed parameter values (if no parameters are fixed this is completely ignored) * @param transforms Array of ParameterLimitsTransform (which can be the NullTransform which does NOT transform the parameter) which transform * a constrained function parameter (e.g. must be between -1 and 1) to a unconstrained fit parameter. * @param fixed BitSet with an element set to <b>true</b> if that parameter is fixed *///w w w. j a v a2 s . com public UncoupledParameterTransforms(final DoubleMatrix1D startValues, final ParameterLimitsTransform[] transforms, final BitSet fixed) { ArgumentChecker.notNull(startValues, "null start values"); ArgumentChecker.notEmpty(transforms, "must specify transforms"); ArgumentChecker.notNull(fixed, "must specify what is fixed (even if none)"); _nMP = startValues.getNumberOfElements(); ArgumentChecker.isTrue(_nMP == transforms.length, "Have {}-dimensional start value but {} transforms", _nMP, transforms.length); _freeParameters = new boolean[_nMP]; for (int i = 0; i < _nMP; i++) { if (i < fixed.size()) { _freeParameters[i] = !fixed.get(i); } else { _freeParameters[i] = true; } } final int count = fixed.cardinality(); ArgumentChecker.isTrue(count < _nMP, "all parameters are fixed"); _nFP = _nMP - count; _startValues = startValues; _transforms = transforms; }
From source file:org.apache.tez.runtime.library.common.writers.UnorderedPartitionedKVWriter.java
private Event generateEvent() throws IOException { DataMovementEventPayloadProto.Builder payloadBuidler = DataMovementEventPayloadProto.newBuilder(); String host = getHost();//from ww w . j av a 2 s .co m int shufflePort = getShufflePort(); BitSet emptyPartitions = new BitSet(); for (int i = 0; i < numPartitions; i++) { if (numRecordsPerPartition[i] == 0) { emptyPartitions.set(i); } } if (emptyPartitions.cardinality() != 0) { // Empty partitions exist ByteString emptyPartitionsByteString = TezCommonUtils .compressByteArrayToByteString(TezUtilsInternal.toByteArray(emptyPartitions)); payloadBuidler.setEmptyPartitions(emptyPartitionsByteString); } if (emptyPartitions.cardinality() != numPartitions) { // Populate payload only if at least 1 partition has data payloadBuidler.setHost(host); payloadBuidler.setPort(shufflePort); payloadBuidler.setPathComponent(outputContext.getUniqueIdentifier()); } CompositeDataMovementEvent cDme = CompositeDataMovementEvent.create(0, numPartitions, payloadBuidler.build().toByteString().asReadOnlyByteBuffer()); return cDme; }