List of usage examples for java.util BitSet BitSet
private BitSet(long[] words)
From source file:bes.injector.InjectorBurnTest.java
private void testPromptnessOfExecution(long intervalNanos, float loadIncrement) throws InterruptedException, ExecutionException, TimeoutException { final int executorCount = 4; int threadCount = 8; int maxQueued = 1024; final WeibullDistribution workTime = new WeibullDistribution(3, 200000); final long minWorkTime = TimeUnit.MICROSECONDS.toNanos(1); final long maxWorkTime = TimeUnit.MILLISECONDS.toNanos(1); final int[] threadCounts = new int[executorCount]; final WeibullDistribution[] workCount = new WeibullDistribution[executorCount]; final ExecutorService[] executors = new ExecutorService[executorCount]; final Injector injector = new Injector(""); for (int i = 0; i < executors.length; i++) { executors[i] = injector.newExecutor(threadCount, maxQueued); threadCounts[i] = threadCount;/*from w ww . ja v a 2 s . c o m*/ workCount[i] = new WeibullDistribution(2, maxQueued); threadCount *= 2; maxQueued *= 2; } long runs = 0; long events = 0; final TreeSet<Batch> pending = new TreeSet<Batch>(); final BitSet executorsWithWork = new BitSet(executorCount); long until = 0; // basic idea is to go through different levels of load on the executor service; initially is all small batches // (mostly within max queue size) of very short operations, moving to progressively larger batches // (beyond max queued size), and longer operations for (float multiplier = 0f; multiplier < 2.01f;) { if (System.nanoTime() > until) { System.out.println(String.format("Completed %.0fK batches with %.1fM events", runs * 0.001f, events * 0.000001f)); events = 0; until = System.nanoTime() + intervalNanos; multiplier += loadIncrement; System.out.println(String.format("Running for %ds with load multiplier %.1f", TimeUnit.NANOSECONDS.toSeconds(intervalNanos), multiplier)); } // wait a random amount of time so we submit new tasks in various stages of long timeout; if (pending.isEmpty()) timeout = 0; else if (Math.random() > 0.98) timeout = Long.MAX_VALUE; else if (pending.size() == executorCount) timeout = pending.first().timeout; else timeout = (long) (Math.random() * pending.last().timeout); while (!pending.isEmpty() && timeout > System.nanoTime()) { Batch first = pending.first(); boolean complete = false; try { for (Result result : first.results.descendingSet()) result.future.get(timeout - System.nanoTime(), TimeUnit.NANOSECONDS); complete = true; } catch (TimeoutException e) { } if (!complete && System.nanoTime() > first.timeout) { for (Result result : first.results) if (!result.future.isDone()) throw new AssertionError(); complete = true; } if (complete) { pending.pollFirst(); executorsWithWork.clear(first.executorIndex); } } // if we've emptied the executors, give all our threads an opportunity to spin down if (timeout == Long.MAX_VALUE) { try { Thread.sleep(10); } catch (InterruptedException e) { } } // submit a random batch to the first free executor service int executorIndex = executorsWithWork.nextClearBit(0); if (executorIndex >= executorCount) continue; executorsWithWork.set(executorIndex); ExecutorService executor = executors[executorIndex]; TreeSet<Result> results = new TreeSet<Result>(); int count = (int) (workCount[executorIndex].sample() * multiplier); long targetTotalElapsed = 0; long start = System.nanoTime(); long baseTime; if (Math.random() > 0.5) baseTime = 2 * (long) (workTime.sample() * multiplier); else baseTime = 0; for (int j = 0; j < count; j++) { long time; if (baseTime == 0) time = (long) (workTime.sample() * multiplier); else time = (long) (baseTime * Math.random()); if (time < minWorkTime) time = minWorkTime; if (time > maxWorkTime) time = maxWorkTime; targetTotalElapsed += time; Future<?> future = executor.submit(new WaitTask(time)); results.add(new Result(future, System.nanoTime() + time)); } long end = start + (long) Math.ceil(targetTotalElapsed / (double) threadCounts[executorIndex]) + TimeUnit.MILLISECONDS.toNanos(100L); long now = System.nanoTime(); if (runs++ > executorCount && now > end) throw new AssertionError(); events += results.size(); pending.add(new Batch(results, end, executorIndex)); // System.out.println(String.format("Submitted batch to executor %d with %d items and %d permitted millis", executorIndex, count, TimeUnit.NANOSECONDS.toMillis(end - start))); } }
From source file:org.apache.cassandra.concurrent.LongSharedExecutorPoolTest.java
private void testPromptnessOfExecution(long intervalNanos, float loadIncrement) throws InterruptedException, ExecutionException { final int executorCount = 4; int threadCount = 8; int maxQueued = 1024; final WeibullDistribution workTime = new WeibullDistribution(3, 200000); final long minWorkTime = TimeUnit.MICROSECONDS.toNanos(1); final long maxWorkTime = TimeUnit.MILLISECONDS.toNanos(1); final int[] threadCounts = new int[executorCount]; final WeibullDistribution[] workCount = new WeibullDistribution[executorCount]; final ExecutorService[] executors = new ExecutorService[executorCount]; for (int i = 0; i < executors.length; i++) { executors[i] = SharedExecutorPool.SHARED.newExecutor(threadCount, maxQueued, "test" + i, "test" + i); threadCounts[i] = threadCount;/*from w w w. j av a2s . c om*/ workCount[i] = new WeibullDistribution(2, maxQueued); threadCount *= 2; maxQueued *= 2; } long runs = 0; long events = 0; final TreeSet<Batch> pending = new TreeSet<>(); final BitSet executorsWithWork = new BitSet(executorCount); long until = 0; // basic idea is to go through different levels of load on the executor service; initially is all small batches // (mostly within max queue size) of very short operations, moving to progressively larger batches // (beyond max queued size), and longer operations for (float multiplier = 0f; multiplier < 2.01f;) { if (System.nanoTime() > until) { System.out.println(String.format("Completed %.0fK batches with %.1fM events", runs * 0.001f, events * 0.000001f)); events = 0; until = System.nanoTime() + intervalNanos; multiplier += loadIncrement; System.out.println(String.format("Running for %ds with load multiplier %.1f", TimeUnit.NANOSECONDS.toSeconds(intervalNanos), multiplier)); } // wait a random amount of time so we submit new tasks in various stages of long timeout; if (pending.isEmpty()) timeout = 0; else if (Math.random() > 0.98) timeout = Long.MAX_VALUE; else if (pending.size() == executorCount) timeout = pending.first().timeout; else timeout = (long) (Math.random() * pending.last().timeout); while (!pending.isEmpty() && timeout > System.nanoTime()) { Batch first = pending.first(); boolean complete = false; try { for (Result result : first.results.descendingSet()) result.future.get(timeout - System.nanoTime(), TimeUnit.NANOSECONDS); complete = true; } catch (TimeoutException e) { } if (!complete && System.nanoTime() > first.timeout) { for (Result result : first.results) if (!result.future.isDone()) throw new AssertionError(); complete = true; } if (complete) { pending.pollFirst(); executorsWithWork.clear(first.executorIndex); } } // if we've emptied the executors, give all our threads an opportunity to spin down if (timeout == Long.MAX_VALUE) Uninterruptibles.sleepUninterruptibly(10, TimeUnit.MILLISECONDS); // submit a random batch to the first free executor service int executorIndex = executorsWithWork.nextClearBit(0); if (executorIndex >= executorCount) continue; executorsWithWork.set(executorIndex); ExecutorService executor = executors[executorIndex]; TreeSet<Result> results = new TreeSet<>(); int count = (int) (workCount[executorIndex].sample() * multiplier); long targetTotalElapsed = 0; long start = System.nanoTime(); long baseTime; if (Math.random() > 0.5) baseTime = 2 * (long) (workTime.sample() * multiplier); else baseTime = 0; for (int j = 0; j < count; j++) { long time; if (baseTime == 0) time = (long) (workTime.sample() * multiplier); else time = (long) (baseTime * Math.random()); if (time < minWorkTime) time = minWorkTime; if (time > maxWorkTime) time = maxWorkTime; targetTotalElapsed += time; Future<?> future = executor.submit(new WaitTask(time)); results.add(new Result(future, System.nanoTime() + time)); } long end = start + (long) Math.ceil(targetTotalElapsed / (double) threadCounts[executorIndex]) + TimeUnit.MILLISECONDS.toNanos(100L); long now = System.nanoTime(); if (runs++ > executorCount && now > end) throw new AssertionError(); events += results.size(); pending.add(new Batch(results, end, executorIndex)); // System.out.println(String.format("Submitted batch to executor %d with %d items and %d permitted millis", executorIndex, count, TimeUnit.NANOSECONDS.toMillis(end - start))); } }
From source file:com.cloudera.oryx.kmeans.computation.cluster.KSketchIndex.java
private BitSet index(RealVector vec) { final double[] prod = new double[projectionBits]; vec.walkInDefaultOrder(new AbstractRealVectorPreservingVisitor() { @Override/*from w ww . java 2 s .c o m*/ public void visit(int index, double value) { for (int j = 0; j < projectionBits; j++) { prod[j] += value * projection[index + j * dimensions]; } } }); BitSet bitset = new BitSet(projectionBits); for (int i = 0; i < projectionBits; i++) { if (prod[i] > 0.0) { bitset.set(i); } } return bitset; }
From source file:org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.java
public int updateBlockReportContext(BlockReportContext context) { if (curBlockReportId != context.getReportId()) { curBlockReportId = context.getReportId(); curBlockReportRpcsSeen = new BitSet(context.getTotalRpcs()); }//www .j a v a2 s . com curBlockReportRpcsSeen.set(context.getCurRpc()); return curBlockReportRpcsSeen.cardinality(); }
From source file:eu.crisis_economics.abm.markets.clearing.heterogeneous.MixedClearingNetworkNode.java
/** * Update (recalculate) the response of this node to a * given connecting edge.// ww w. j a va 2 s .c o m */ @Override public final void flagEdgeResponseForUpdate(final SingletonEdge edge) { final int connectionIndex = getConnectionIndexOfEdge(edge); if (knownEdgeRates == null) { knownEdgeRates = collectEdgeRates(); edgeRateIsUpdated = new BitSet(knownEdgeRates.length); edgeRateIsUpdated.flip(0, knownEdgeRates.length); } else { knownEdgeRates[connectionIndex].setRate(getEdgeRate(edge)); edgeRateIsUpdated.set(connectionIndex); } }
From source file:com.eclecticlogic.pedal.TestPedalDialect.java
@Test @Transactional//from w w w .java2 s .c om public void testCopyCommand() { CopyList<ExoticTypes> list = new CopyList<>(); // The copy-command can insert 100k of these per second. for (int i = 0; i < 10; i++) { ExoticTypes et = new ExoticTypes(); et.setLogin("copyCommand" + i); BitSet bs = new BitSet(7); bs.set(1); bs.set(3); bs.set(4); et.setCountries(bs); et.setAuthorizations(Sets.newHashSet("a", "b", "b", "c")); if (i != 9) { et.setScores(Lists.newArrayList(1L, 2L, 3L)); } else { et.setScores(Lists.<Long>newArrayList()); } et.setStatus(Status.ACTIVE); et.setCustom("this will be made uppercase"); list.add(et); } copyCommand.insert(entityManager, list); Assert.assertNotNull(entityManager.find(ExoticTypes.class, "copyCommand0")); Assert.assertEquals(entityManager.find(ExoticTypes.class, "copyCommand0").getCustom(), "THIS WILL BE MADE UPPERCASE"); Assert.assertNotNull(entityManager.find(ExoticTypes.class, "copyCommand1")); Assert.assertEquals(entityManager.find(ExoticTypes.class, "copyCommand0").getAuthorizations(), Sets.newHashSet("b", "c", "a")); }
From source file:edu.oregonstate.eecs.mcplan.domains.tetris.TetrisState.java
public TetrisState(final TetrisParameters params) { this.params = params; // cells = new byte[params.Nrows][params.Ncolumns]; // cells = new ArrayList<BitSet>( params.Nrows ); // for( int i = 0; i < params.Nrows; ++i ) { // cells.set( i, new BitSet( params.Ncolumns ) ); // }// www . java2 s . c o m cells = new BitSet(params.Nrows * params.Ncolumns); }
From source file:fingerprints.helper.BloomFilter.java
/** * You must specify the number of bits in the Bloom Filter, and also you should specify the number of items you * expect to add./*from w ww . j a v a2 s . c o m*/ * * The latter is used to choose some optimal internal values to minimize the false-positive rate (which can be * estimated with expectedFalsePositiveRate()). * * @param bisetSize The number of bits in the bit array (often called 'm' in the context of bloom filters). * @param expectedPatterns The typical number of items you expect to be added to the SimpleBloomFilter (often called * 'n'). */ public BloomFilter(int bisetSize, int expectedPatterns) { this.bitSetSize = bisetSize; this.expectedPatterns = expectedPatterns; // k = ceil(-log_2(false prob.)) double falsePositiveProbability = (bisetSize / expectedPatterns); this.k = (int) Math.ceil((falsePositiveProbability) * Math.log(2.0)); bitSet = new BitSet(bisetSize); }
From source file:com.surevine.alfresco.esl.impl.ESCAfterInvocationProvider.java
/** * Called from decide(...) this method filters the contents of the input ResultSet, returning an output ResultSet containing only those results that RMCaveatConfig allows the current user to see * // w w w. j a v a 2 s .c om * @param unfilteredResultSet * A result set to filter * @return ResultSet containing only those elemenets in <code>unfilteredResultSet</code> for which the Caveat Service allows access to the current user */ protected ResultSet decideOnResultSet(ResultSet unfilteredResultSet) { boolean recordAnyMissingNodes = new Date().after(_nextTimeToReportMissingNodes); boolean foundMissingNodes = false; if (LOGGER.isDebugEnabled()) { LOGGER.debug("Entering decideOnResultSet"); } // Shortcut some obvious exit conditions if (unfilteredResultSet == null) { return null; } if (unfilteredResultSet.length() == 0) { return unfilteredResultSet; } BitSet inclusionMask = new BitSet(unfilteredResultSet.length()); FilteringResultSet frs = new FilteringResultSet(unfilteredResultSet, inclusionMask); int length = unfilteredResultSet.length(); for (int i = 0; i < length; i++) { NodeRef nodeRef = unfilteredResultSet.getNodeRef(i); if (_nodeService.exists(nodeRef)) { // If the node exists, check whether we can see it if (_caveatComponent.hasAccess(nodeRef)) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Access Granted to " + nodeRef); } inclusionMask.set(i); } else if (LOGGER.isDebugEnabled()) { LOGGER.debug("Access forbidden to " + nodeRef); } } else { foundMissingNodes = true; if (recordAnyMissingNodes) { LOGGER.warn("The node [" + nodeRef + "] was returned from a search but does not exist."); } } } if (foundMissingNodes) { _nextTimeToReportMissingNodes = new Date(new Date().getTime() + _missingNodeReportingFrequencyMillis); LOGGER.info("To preserve performance, the system will not report on further missing nodes until " + _nextTimeToReportMissingNodes); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Leaving decideOnResultSet"); } return frs; }
From source file:org.apache.lucene.index.collocations.CollocationExtractor.java
/** * Called for every term in the index//from w w w.j av a 2s . co m * docsAndPositions, possible speed up by http://lucene.apache.org/core/4_2_0/core/org/apache/lucene/index/TermsEnum.html * http://stackoverflow.com/questions/15771843/get-word-position-in-document-with-lucene * Migration Guide: http://lucene.apache.org/core/4_8_1/MIGRATE.html * http://stackoverflow.com/questions/15370652/retrieving-all-term-positions-from-docsandpositionsenum * @param bytesRef * @param logger * @param slop * @throws IOException */ void processTerm(BytesRef bytesRef, CollocationIndexer logger, int slop) throws IOException { Term term = new Term(this.fieldName, bytesRef); if (!filter.processTerm(term.text())) { return; } System.out.println("Processing term: " + term); // TermEnum te = reader.terms(term); // int numDocsForTerm = Math.min(te.docFreq(), maxNumDocsToAnalyze); int numDocsForTerm = Math.min(this.reader.docFreq(term), maxNumDocsToAnalyze); int totalNumDocs = reader.numDocs(); float percent = (float) numDocsForTerm / (float) totalNumDocs; isTermTooPopularOrNotPopularEnough(term, percent); // get a list of all the docs with this term // Apache Lucene Migration Guide // TermDocs td = reader.termDocs(term); // get dpe in first hand DocsAndPositionsEnum dpe = MultiFields.getTermPositionsEnum(this.reader, null, this.fieldName, bytesRef); HashMap<String, CollocationScorer> phraseTerms = new HashMap<String, CollocationScorer>(); int MAX_TERMS_PER_DOC = 100000; BitSet termPos = new BitSet(MAX_TERMS_PER_DOC); int numDocsAnalyzed = 0; // for all docs that contain this term int docSeq; while ((docSeq = dpe.nextDoc()) != DocsEnum.NO_MORE_DOCS) { int docId = dpe.docID(); // System.out.println("Processing docId: "+docId); numDocsAnalyzed++; if (numDocsAnalyzed > maxNumDocsToAnalyze) { break; } // get TermPositions for matching doc // TermPositionVector tpv = (TermPositionVector) reader.getTermFreqVector(docId, fieldName); // String[] terms_str = tpv.getTerms(); Terms tv = this.reader.getTermVector(docId, this.fieldName); TermsEnum te = tv.iterator(null); // TODO refactor iteration List<String> terms_list = new ArrayList<>(); while (te.next() != null) { terms_list.add(te.term().utf8ToString()); } String[] terms_str = terms_list.toArray(new String[terms_list.size()]); // System.out.println("terms_str: "+Arrays.toString(terms_str)); termPos.clear(); int index = recordAllPositionsOfTheTermInCurrentDocumentBitset(docSeq, term, termPos, tv, terms_str); // now look at all OTHER terms_str in this doc and see if they are // positioned in a pre-defined sized window around the current term /* for (int j = 0; j < terms_str.length; j++) { if (j == index) { // (item A) continue; } if (!filter.processTerm(terms_str[j])) { continue; } if (!StringUtils.isAlpha(terms_str[j])) { continue; } // sequential code boolean matchFound = false; for (int k = 0; ((k < dpe.freq()) && (!matchFound)); k++) { try { // inefficient // iterate through all other items (item B) Integer position = dpe.nextPosition(); Integer startpos = Math.max(0, position - slop); Integer endpos = position + slop; matchFound = populateHashMapWithPhraseTerms(term, numDocsForTerm, totalNumDocs, phraseTerms, termPos, terms_str, j, matchFound, startpos, endpos); } catch (ArrayIndexOutOfBoundsException e) { e.printStackTrace(); break; } catch (IOException e) { e.printStackTrace(); break; } } } */ /// boolean[] matchFound = new boolean[terms_str.length]; // single match is sufficient, no duplicate process for (int j = 0; j < matchFound.length; j++) matchFound[j] = false; for (int k = 0; (k < dpe.freq()); k++) { Integer position = dpe.nextPosition(); Integer startpos = Math.max(0, position - slop); Integer endpos = position + slop; for (int j = 0; j < terms_str.length && !matchFound[j]; j++) { if (j == index) { // (item A) continue; } if (!filter.processTerm(terms_str[j])) { continue; } if (!StringUtils.isAlpha(terms_str[j])) { continue; } // inefficient // iterate through all other items (item B) populateHashMapWithPhraseTerms(term, numDocsForTerm, totalNumDocs, phraseTerms, termPos, terms_str, j, matchFound, startpos, endpos); } } } // end docs loop sortTopTermsAndAddToCollocationsIndexForThisTerm(logger, phraseTerms); }