List of usage examples for java.util TreeMap size
int size
To view the source code for java.util TreeMap size.
Click Source Link
From source file:com.datatorrent.contrib.hdht.HDHTWriter.java
/** * Write out the changes to the file because of purge operation. If any purge range completely * covers the keys in the file then delete the file without even opening and reading the keys * from the file./*w w w .ja v a2s . c om*/ * * remove keys overlapping with purge ranges and they write out to a new file. * @param bucket bucket * @param meta file meta data. * @param rset purge range set which overlaps with the file. * @param filesToDelete if file is being deleted completely then add it to this list. * @param bmeta bucket metadata. * @throws IOException */ private void writeFileWithPurge(Bucket bucket, BucketFileMeta meta, RangeSet<Slice> rset, HashSet<String> filesToDelete, BucketMeta bmeta) throws IOException { LOG.debug("Writing file because of purge operation {}", meta); if (rset.containsFully(new Range<>(meta.startKey, getEndKey(bucket.bucketKey, meta, rset)))) { LOG.info("File being deleted because of purge {}", meta); filesToDelete.add(meta.name); bmeta.files.remove(meta.startKey); return; } TreeMap<Slice, Slice> fileData = readDataExcludingPurge(bucket, meta, rset); /** Rewrite the file, if any key from file is removed as part of purge, * and there is some data to be written. */ if (fileData.size() > 0) { LOG.info("Rewriting file because of purge {}", meta); filesToDelete.add(meta.name); bmeta.files.remove(meta.startKey); writeFile(bucket, bmeta, fileData); } }
From source file:org.loklak.geo.GeoNames.java
public LinkedHashSet<String> suggest(String q, int count, int distance) { TreeMap<Long, String> a = new TreeMap<>(); String ql = normalize(q);//from w w w .ja v a2 s . com boolean exact = false; String exactTerm = null; seekloop: for (GeoLocation g : id2loc.values()) { termloop: for (String n : g.getNames()) { if (n.length() > 3 && n.length() < ql.length() * 4) { String nn = normalize(n); if (!exact && nn.equals(ql)) { exact = true; exactTerm = n; continue seekloop; } // starts-with: if (nn.startsWith(ql)) { a.put(g.getPopulation() + a.size(), n); if (a.size() > count * 2) break seekloop; } // distance if (nn.length() == ql.length()) { int errorcount = 0; for (int i = 0; i < nn.length(); i++) { if (nn.charAt(i) != ql.charAt(i)) { errorcount++; if (errorcount > distance) continue termloop; } } a.put(g.getPopulation() + a.size(), n); if (a.size() > count * 2) break seekloop; } } } } // order by population LinkedHashSet<String> list = new LinkedHashSet<>(); int i = 0; if (exact) { list.add(exactTerm); } for (Long p : a.descendingKeySet()) { list.add(a.get(p)); if (i >= list.size()) break; } return list; }
From source file:org.apache.hadoop.hbase.master.handler.TableEventHandler.java
public boolean reOpenAllRegions(List<HRegionInfo> regions) throws IOException { boolean done = false; LOG.info("Bucketing regions by region server..."); HTable table = new HTable(masterServices.getConfiguration(), tableName); TreeMap<ServerName, List<HRegionInfo>> serverToRegions = Maps.newTreeMap(); NavigableMap<HRegionInfo, ServerName> hriHserverMapping; try {//www . j a v a 2 s . c o m hriHserverMapping = table.getRegionLocations(); } finally { table.close(); } List<HRegionInfo> reRegions = new ArrayList<HRegionInfo>(); for (HRegionInfo hri : regions) { ServerName rsLocation = hriHserverMapping.get(hri); // Skip the offlined split parent region // See HBASE-4578 for more information. if (null == rsLocation) { LOG.info("Skip " + hri); continue; } if (!serverToRegions.containsKey(rsLocation)) { LinkedList<HRegionInfo> hriList = Lists.newLinkedList(); serverToRegions.put(rsLocation, hriList); } reRegions.add(hri); serverToRegions.get(rsLocation).add(hri); } LOG.info("Reopening " + reRegions.size() + " regions on " + serverToRegions.size() + " region servers."); this.masterServices.getAssignmentManager().setRegionsToReopen(reRegions); BulkReOpen bulkReopen = new BulkReOpen(this.server, serverToRegions, this.masterServices.getAssignmentManager()); while (true) { try { if (bulkReopen.bulkReOpen()) { done = true; break; } else { LOG.warn("Timeout before reopening all regions"); } } catch (InterruptedException e) { LOG.warn("Reopen was interrupted"); // Preserve the interrupt. Thread.currentThread().interrupt(); break; } } return done; }
From source file:com.vuze.android.remote.dialog.DialogFragmentFilterByTags.java
@NonNull @Override// w w w .j a va 2s .c om public Dialog onCreateDialog(Bundle savedInstanceState) { SessionInfo sessionInfo = getSessionInfo(); List<Map<?, ?>> tags = sessionInfo == null ? null : sessionInfo.getTags(); if (tags != null && tags.size() > 0) { TreeMap<String, Long> map = new TreeMap<>(); for (Object o : tags) { if (o instanceof Map) { Map<?, ?> mapTag = (Map<?, ?>) o; long uid = MapUtils.getMapLong(mapTag, "uid", 0); String name = MapUtils.getMapString(mapTag, "name", "??"); int type = MapUtils.getMapInt(mapTag, "type", 0); if (type == 3) { // type-name will be "Manual" :( name = "Tag: " + name; } else { String typeName = MapUtils.getMapString(mapTag, "type-name", null); if (typeName != null) { name = typeName + ": " + name; } } map.put(name, uid); } } long[] vals = new long[map.size()]; String[] strings = map.keySet().toArray(new String[map.keySet().size()]); for (int i = 0; i < vals.length; i++) { vals[i] = map.get(strings[i]); } filterByList = new ValueStringArray(vals, strings); } if (filterByList == null) { filterByList = AndroidUtils.getValueStringArray(getResources(), R.array.filterby_list); } AndroidUtils.AlertDialogBuilder alertDialogBuilder = AndroidUtils.createAlertDialogBuilder(getActivity(), R.layout.dialog_filter_by); View view = alertDialogBuilder.view; AlertDialog.Builder builder = alertDialogBuilder.builder; // get our tabHost from the xml TabHost tabHost = (TabHost) view.findViewById(R.id.filterby_tabhost); tabHost.setup(); // create tab 1 TabHost.TabSpec spec1 = tabHost.newTabSpec("tab1"); spec1.setIndicator("States"); spec1.setContent(R.id.filterby_sv_state); tabHost.addTab(spec1); //create tab2 TabHost.TabSpec spec2 = tabHost.newTabSpec("tab2"); spec2.setIndicator("Tags"); spec2.setContent(R.id.filterby_tv_tags); tabHost.addTab(spec2); int height = AndroidUtilsUI.dpToPx(32); tabHost.getTabWidget().getChildAt(0).getLayoutParams().height = height; tabHost.getTabWidget().getChildAt(1).getLayoutParams().height = height; TextView tvState = (TextView) view.findViewById(R.id.filterby_tv_state); tvState.setMovementMethod(LinkMovementMethod.getInstance()); final TextView tvTags = (TextView) view.findViewById(R.id.filterby_tv_tags); tvTags.setMovementMethod(LinkMovementMethod.getInstance()); // for API <= 10 (maybe 11?), otherwise tags will display on one line tabHost.setOnTabChangedListener(new TabHost.OnTabChangeListener() { @Override public void onTabChanged(String tabId) { if (!tabId.equals("tab2")) { return; } tvTags.post(new Runnable() { @Override public void run() { spanTags.updateTags(); } }); } }); builder.setTitle(R.string.filterby_title); // Add action buttons builder.setPositiveButton(R.string.action_filterby, new DialogInterface.OnClickListener() { @Override public void onClick(DialogInterface dialog, int id) { if (mapSelectedTag == null) { return; } long uidSelected = MapUtils.getMapLong(mapSelectedTag, "uid", -1); String name = MapUtils.getMapString(mapSelectedTag, "name", "??"); mListener.filterBy(uidSelected, name, true); } }); builder.setNegativeButton(android.R.string.cancel, new DialogInterface.OnClickListener() { @Override public void onClick(DialogInterface dialog, int id) { DialogFragmentFilterByTags.this.getDialog().cancel(); } }); List<Map<?, ?>> manualTags = new ArrayList<>(); List<Map<?, ?>> stateTags = new ArrayList<>(); if (sessionInfo != null) { // Dialog never gets called wehn getTags has no tags List<Map<?, ?>> allTags = sessionInfo.getTags(); if (allTags != null) { for (Map<?, ?> mapTag : allTags) { int type = MapUtils.getMapInt(mapTag, "type", 0); switch (type) { case 0: case 1: case 2: stateTags.add(mapTag); break; case 3: // manual manualTags.add(mapTag); break; } } } } SpanTagsListener l = new SpanTagsListener() { @Override public void tagClicked(Map mapTag, String name) { mapSelectedTag = mapTag; // todo: long click, don't exit long uidSelected = MapUtils.getMapLong(mapSelectedTag, "uid", -1); mListener.filterBy(uidSelected, name, true); DialogFragmentFilterByTags.this.getDialog().dismiss(); } @Override public int getTagState(Map mapTag, String name) { if (mapSelectedTag == null) { return SpanTags.TAG_STATE_UNSELECTED; } long uidSelected = MapUtils.getMapLong(mapSelectedTag, "uid", -1); if (uidSelected == -1) { return SpanTags.TAG_STATE_UNSELECTED; } long uidQuery = MapUtils.getMapLong(mapTag, "uid", -1); return uidQuery == uidSelected ? SpanTags.TAG_STATE_SELECTED : SpanTags.TAG_STATE_UNSELECTED; } }; spanTags = new SpanTags(getActivity(), sessionInfo, tvTags, l); spanTags.setTagMaps(manualTags); spanTags.setShowIcon(false); spanTags.updateTags(); SpanTags spanState = new SpanTags(getActivity(), sessionInfo, tvState, l); spanState.setTagMaps(stateTags); spanState.setShowIcon(false); spanState.updateTags(); return builder.create(); }
From source file:org.nlp2rdf.implementation.spotlight.SpotlightWrapper.java
public void processText(Individual context, OntModel inputModel, OntModel outputModel, NIFParameters nifParameters) {/*from ww w .j a va2s . c om*/ String contextString = context .getPropertyValue(NIFDatatypeProperties.isString.getDatatypeProperty(inputModel)).asLiteral() .getString(); String prefix = nifParameters.getPrefix(); URIScheme urischeme = nifParameters.getUriScheme(); confidence = nifParameters.getOptions().valueOf("confidence").toString(); Properties props = new Properties(); props.put("annotators", "tokenize, ssplit"); StanfordCoreNLP pipeline = new StanfordCoreNLP(props); // create an empty Annotation just with the given text Annotation document = new Annotation(contextString); // run all Annotators on this text pipeline.annotate(document); // these are all the sentences in this document // a CoreMap is essentially a Map that uses class objects as keys and // has values with custom types List<CoreMap> sentences = document.get(CoreAnnotations.SentencesAnnotation.class); // get all the sentences and words and read it in an intermediate // structure // NOTE: this can be greatly optimized of course // for now it is just simple and cheap to implement it like this int wordCount = 0; TreeMap<Span, List<Span>> tokenizedText = new TreeMap<Span, List<Span>>(); for (CoreMap sentence : sentences) { Span sentenceSpan = new Span(sentence.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class), sentence.get(CoreAnnotations.CharacterOffsetEndAnnotation.class)); List<Span> wordSpans = new ArrayList<Span>(); for (CoreLabel coreLabel : sentence.get(CoreAnnotations.TokensAnnotation.class)) { wordSpans.add(new Span(coreLabel.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class), coreLabel.get(CoreAnnotations.CharacterOffsetEndAnnotation.class))); wordCount++; } tokenizedText.put(sentenceSpan, wordSpans); } Text2RDF t = new Text2RDF(); t.generateNIFModel(prefix, context, urischeme, inputModel, tokenizedText); outputModel.add(RLOGSLF4JBinding.log(nifParameters.getLogPrefix(), "Finished creating " + tokenizedText.size() + " sentence(s) with " + wordCount + " word(s) ", RLOGIndividuals.DEBUG, this.getClass().getCanonicalName(), null, null)); // query spotlight querySpotlight(contextString); // traversing the words in the current sentence // a CoreLabel is a CoreMap with additional token-specific methods for (CoreMap sentence : sentences) { for (CoreLabel token : sentence.get(CoreAnnotations.TokensAnnotation.class)) { Span wordSpan = new Span(token.get(CoreAnnotations.CharacterOffsetBeginAnnotation.class), token.get(CoreAnnotations.CharacterOffsetEndAnnotation.class)); // the word should exist already Individual wordIndividual = outputModel .getIndividual(urischeme.generate(prefix, contextString, wordSpan)); if (wordIndividual == null) { log.error("SKIPPING: word was not found in the model: " + urischeme.generate(prefix, contextString, wordSpan)); continue; } if (h.get(String.valueOf(wordSpan.getStart())) != null) { List<String> l = h.get(String.valueOf(wordSpan.getStart())); Iterator<String> iterator = l.iterator(); while (iterator.hasNext()) { String uri = iterator.next(); wordIndividual.addProperty( outputModel.getProperty("http://www.w3.org/2005/11/its/rdf#", "taIdentRef"), outputModel.createResource(dbpediaResourceLink + uri)); } } outputModel.setNsPrefix("itsrdf", "http://www.w3.org/2005/11/its/rdf#"); } } }
From source file:org.loklak.geo.GeoNames.java
public GeoNames(final File cities1000_zip, final File iso3166json, long minPopulation) throws IOException { // load iso3166 info this.iso3166toCountry = new HashMap<>(); try {//from w w w. j a va 2 s . co m //String jsonString = new String(Files.readAllBytes(iso3166json.toPath()), StandardCharsets.UTF_8); ObjectMapper jsonMapper = new ObjectMapper(DAO.jsonFactory); JsonNode j = jsonMapper.readTree(iso3166json); for (JsonNode n : j) { // contains name,alpha-2,alpha-3,country-code,iso_3166-2,region-code,sub-region-code String name = n.get("name").textValue(); String cc = n.get("alpha-2").textValue(); this.iso3166toCountry.put(cc, name); } } catch (IOException e) { this.iso3166toCountry = new HashMap<String, String>(); } // this is a processing of the cities1000.zip file from http://download.geonames.org/export/dump/ this.id2loc = new HashMap<>(); this.hash2ids = new HashMap<>(); this.stopwordHashes = new HashSet<>(); this.countryCenter = new HashMap<>(); Map<String, CountryBounds> countryBounds = new HashMap<>(); if (cities1000_zip == null || !cities1000_zip.exists()) { throw new IOException("GeoNames: file does not exist!"); } ZipFile zf = null; BufferedReader reader = null; try { zf = new ZipFile(cities1000_zip); String entryName = cities1000_zip.getName(); entryName = entryName.substring(0, entryName.length() - 3) + "txt"; final ZipEntry ze = zf.getEntry(entryName); final InputStream is = zf.getInputStream(ze); reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); } catch (final IOException e) { throw new IOException("GeoNames: Error when decompressing cities1000.zip!", e); } /* parse this fields: --------------------------------------------------- 00 geonameid : integer id of record in geonames database 01 name : name of geographical point (utf8) varchar(200) 02 asciiname : name of geographical point in plain ascii characters, varchar(200) 03 alternatenames : alternatenames, comma separated varchar(5000) 04 latitude : latitude in decimal degrees (wgs84) 05 longitude : longitude in decimal degrees (wgs84) 06 feature class : see http://www.geonames.org/export/codes.html, char(1) 07 feature code : see http://www.geonames.org/export/codes.html, varchar(10) 08 country code : ISO-3166 2-letter country code, 2 characters 09 cc2 : alternate country codes, comma separated, ISO-3166 2-letter country code, 60 characters 10 admin1 code : fipscode (subject to change to iso code), see exceptions below, see file admin1Codes.txt for display names of this code; varchar(20) 11 admin2 code : code for the second administrative division, a county in the US, see file admin2Codes.txt; varchar(80) 12 admin3 code : code for third level administrative division, varchar(20) 13 admin4 code : code for fourth level administrative division, varchar(20) 14 population : bigint (8 byte int) 15 elevation : in meters, integer 16 dem : digital elevation model, srtm3 or gtopo30, average elevation of 3''x3'' (ca 90mx90m) or 30''x30'' (ca 900mx900m) area in meters, integer. srtm processed by cgiar/ciat. 17 timezone : the timezone id (see file timeZone.txt) varchar(40) 18 modification date : date of last modification in yyyy-MM-dd format */ try { String line; String[] fields; while ((line = reader.readLine()) != null) { if (line.isEmpty()) { continue; } fields = CommonPattern.TAB.split(line); final long population = Long.parseLong(fields[14]); if (minPopulation > 0 && population < minPopulation) continue; final int geonameid = Integer.parseInt(fields[0]); Set<String> locnames = new LinkedHashSet<>(); locnames.add(fields[1]); locnames.add(fields[2]); for (final String s : CommonPattern.COMMA.split(fields[3])) locnames.add(s); ArrayList<String> locnamess = new ArrayList<>(locnames.size()); locnamess.addAll(locnames); String cc = fields[8]; //ISO-3166 final GeoLocation geoLocation = new GeoLocation(Float.parseFloat(fields[4]), Float.parseFloat(fields[5]), locnamess, cc); geoLocation.setPopulation(population); this.id2loc.put(geonameid, geoLocation); for (final String name : locnames) { if (name.length() < 4) continue; String normalized = normalize(name); int lochash = normalized.hashCode(); List<Integer> locs = this.hash2ids.get(lochash); if (locs == null) { locs = new ArrayList<Integer>(1); this.hash2ids.put(lochash, locs); } if (!locs.contains(geonameid)) locs.add(geonameid); } // update the country bounds CountryBounds bounds = countryBounds.get(cc); if (bounds == null) { bounds = new CountryBounds(); countryBounds.put(cc, bounds); } bounds.extend(geoLocation); } if (reader != null) reader.close(); if (zf != null) zf.close(); } catch (final IOException e) { } // calculate the center of the countries for (Map.Entry<String, CountryBounds> country : countryBounds.entrySet()) { this.countryCenter.put(country.getKey(), new double[] { (country.getValue().lon_west - country.getValue().lon_east) / 2.0, (country.getValue().lat_north - country.getValue().lat_south) / 2.0 }); // [longitude, latitude] } // finally create a statistic which names appear very often to have fill-word heuristic TreeMap<Integer, Set<Integer>> stat = new TreeMap<>(); // a mapping from number of occurrences of location name hashes to a set of location name hashes for (Map.Entry<Integer, List<Integer>> entry : this.hash2ids.entrySet()) { int occurrences = entry.getValue().size(); Set<Integer> hashes = stat.get(occurrences); if (hashes == null) { hashes = new HashSet<Integer>(); stat.put(occurrences, hashes); } hashes.add(entry.getKey()); } // we consider 3/4 of this list as fill-word (approx 300): those with the most occurrences int good = stat.size() / 4; Iterator<Map.Entry<Integer, Set<Integer>>> i = stat.entrySet().iterator(); for (int j = 0; j < good; j++) i.next(); // 'eat away' the good entries. while (i.hasNext()) { Set<Integer> morehashes = i.next().getValue(); this.stopwordHashes.addAll(morehashes); } }
From source file:edu.utexas.cs.tactex.MarketManagerTest.java
/** * Testing the private method that learns * bid clearing distributions for the //from w w w. j a v a2s.com * bidding MDP */ @Test public void testRecordTradeResult() { marketManagerService.initialize(brokerContext); int tradeCreationTime = 1; double executionPrice = 2; double executionMWh = 3; marketManagerService.recordTradeResult(tradeCreationTime, 4, executionPrice, executionMWh); marketManagerService.recordTradeResult(tradeCreationTime, 5, executionPrice, executionMWh); marketManagerService.recordTradeResult(tradeCreationTime, 6, executionPrice, executionMWh); marketManagerService.recordTradeResult(tradeCreationTime, 10, executionPrice - 1, executionMWh); marketManagerService.recordTradeResult(tradeCreationTime, 10, executionPrice + 1, executionMWh); // test: trades are recorded @SuppressWarnings("unchecked") TreeMap<Integer, ArrayList<PriceMwhPair>> supportingBidGroups = (TreeMap<Integer, ArrayList<PriceMwhPair>>) ReflectionTestUtils .getField(marketManagerService, "supportingBidGroups"); assertEquals("supportingBidGroups.size", 4, supportingBidGroups.size()); assertEquals("supportingBidGroups.get(4).size", 1, supportingBidGroups.get(4).size()); assertEquals("supportingBidGroups.get(5).size", 1, supportingBidGroups.get(5).size()); assertEquals("supportingBidGroups.get(6).size", 1, supportingBidGroups.get(6).size()); assertEquals("supportingBidGroups.get(10).size", 2, supportingBidGroups.get(10).size()); assertEquals("supportingBidGroups(4,0).price", executionPrice, supportingBidGroups.get(4).get(0).getPricePerMwh(), 1e-6); assertEquals("supportingBidGroups(5,0).price", executionPrice, supportingBidGroups.get(5).get(0).getPricePerMwh(), 1e-6); assertEquals("supportingBidGroups(6,0).price", executionPrice, supportingBidGroups.get(6).get(0).getPricePerMwh(), 1e-6); assertEquals("supportingBidGroups(10,0).price is lowest", executionPrice - 1, supportingBidGroups.get(10).get(0).getPricePerMwh(), 1e-6); assertEquals("supportingBidGroups(10,1).price is higher", executionPrice + 1, supportingBidGroups.get(10).get(1).getPricePerMwh(), 1e-6); assertEquals("supportingBidGroups(4,0).mwh", executionMWh, supportingBidGroups.get(4).get(0).getMwh(), 1e-6); assertEquals("supportingBidGroups(5,0).mwh", executionMWh, supportingBidGroups.get(5).get(0).getMwh(), 1e-6); assertEquals("supportingBidGroups(6,0).mwh", executionMWh, supportingBidGroups.get(6).get(0).getMwh(), 1e-6); assertEquals("supportingBidGroups(10,0).mwh", executionMWh, supportingBidGroups.get(10).get(0).getMwh(), 1e-6); assertEquals("supportingBidGroups(10,1).mwh", executionMWh, supportingBidGroups.get(10).get(1).getMwh(), 1e-6); }
From source file:com.joliciel.jochre.analyser.BeamSearchImageAnalyser.java
public void analyseInternal(JochreImage image) { LOG.debug("Analysing image " + image.getId()); if (currentMonitor != null) { currentMonitor.setCurrentAction("imageMonitor.analysingImage", new Object[] { image.getPage().getIndex() }); }/*from ww w . j a v a2 s .c o m*/ for (LetterGuessObserver observer : observers) { observer.onImageStart(image); } if (totalShapeCount < 0) totalShapeCount = image.getShapeCount(); for (Paragraph paragraph : image.getParagraphs()) { LOG.debug("Analysing paragraph " + paragraph.getIndex() + " (id=" + paragraph.getId() + ")"); List<LetterSequence> holdoverSequences = null; for (RowOfShapes row : paragraph.getRows()) { LOG.debug("Analysing row " + row.getIndex() + " (id=" + row.getId() + ")"); for (GroupOfShapes group : row.getGroups()) { if (group.isSkip()) { LOG.debug("Skipping group " + group.getIndex() + " (id=" + group.getId() + ")"); continue; } LOG.debug("Analysing group " + group.getIndex() + " (id=" + group.getId() + ")"); int width = group.getRight() - group.getLeft() + 1; List<ShapeSequence> shapeSequences = null; if (boundaryDetector != null) { shapeSequences = boundaryDetector.findBoundaries(group); } else { // simply add this groups shape's shapeSequences = new ArrayList<ShapeSequence>(); ShapeSequence shapeSequence = boundaryService.getEmptyShapeSequence(); for (Shape shape : group.getShapes()) shapeSequence.addShape(shape); shapeSequences.add(shapeSequence); } // Perform a beam search to guess the most likely sequence for this word TreeMap<Integer, PriorityQueue<LetterSequence>> heaps = new TreeMap<Integer, PriorityQueue<LetterSequence>>(); // prime a starter heap with the n best shape boundary analyses for this group PriorityQueue<LetterSequence> starterHeap = new PriorityQueue<LetterSequence>(1); for (ShapeSequence shapeSequence : shapeSequences) { LetterSequence emptySequence = this.getLetterGuesserService() .getEmptyLetterSequence(shapeSequence); starterHeap.add(emptySequence); } heaps.put(0, starterHeap); PriorityQueue<LetterSequence> finalHeap = null; while (heaps.size() > 0) { Entry<Integer, PriorityQueue<LetterSequence>> heapEntry = heaps.pollFirstEntry(); if (LOG.isTraceEnabled()) LOG.trace("heap for index: " + heapEntry.getKey().intValue() + ", width: " + width); if (heapEntry.getKey().intValue() == width) { finalHeap = heapEntry.getValue(); break; } PriorityQueue<LetterSequence> previousHeap = heapEntry.getValue(); // limit the breadth to K int maxSequences = previousHeap.size() > this.beamWidth ? this.beamWidth : previousHeap.size(); for (int j = 0; j < maxSequences; j++) { LetterSequence history = previousHeap.poll(); ShapeInSequence shapeInSequence = history.getNextShape(); Shape shape = shapeInSequence.getShape(); if (LOG.isTraceEnabled()) { LOG.trace("Sequence " + history + ", shape: " + shape); } LogUtils.logMemory(LOG); int position = 0; if (Linguistics.getInstance(image.getPage().getDocument().getLocale()) .isLeftToRight()) { position = shape.getRight() - group.getLeft() + 1; } else { position = group.getRight() - shape.getLeft() + 1; } PriorityQueue<LetterSequence> heap = heaps.get(position); if (heap == null) { heap = new PriorityQueue<LetterSequence>(); heaps.put(position, heap); } MONITOR.startTask("guess letter"); try { letterGuesser.guessLetter(shapeInSequence, history); } finally { MONITOR.endTask(); } MONITOR.startTask("heap sort"); try { for (Decision<Letter> letterGuess : shape.getLetterGuesses()) { // leave out very low probability outcomes if (letterGuess.getProbability() > this.minOutcomeWeight) { LetterSequence sequence = this.getLetterGuesserService() .getLetterSequencePlusOne(history); sequence.add(letterGuess.getOutcome()); sequence.addDecision(letterGuess); heap.add(sequence); } // weight big enough to include } // next letter guess for this shape } finally { MONITOR.endTask(); } } // next history in heap } // any more heaps? LetterSequence bestSequence = null; boolean shouldCombineWithHoldover = false; boolean isHoldover = false; MONITOR.startTask("best sequence"); try { List<LetterSequence> finalSequences = new ArrayList<LetterSequence>(); for (int i = 0; i < this.beamWidth; i++) { if (finalHeap.isEmpty()) break; finalSequences.add(finalHeap.poll()); } if (this.getMostLikelyWordChooser() == null) { // most likely sequence is on top of the last heap bestSequence = finalSequences.get(0); } else { // get most likely sequence using lexicon if (holdoverSequences != null) { // we have a holdover from the previous row ending with a dash bestSequence = this.getMostLikelyWordChooser().chooseMostLikelyWord(finalSequences, holdoverSequences, this.beamWidth); shouldCombineWithHoldover = true; } else { // check if this is the last group on the row and could end with a dash boolean shouldBeHeldOver = false; if (group.getIndex() == row.getGroups().size() - 1 && row.getIndex() < paragraph.getRows().size() - 1) { for (LetterSequence letterSequence : finalSequences) { if (letterSequence.toString().endsWith("-")) { shouldBeHeldOver = true; break; } } } if (shouldBeHeldOver) { holdoverSequences = finalSequences; isHoldover = true; } else { // simplest case: no holdover bestSequence = this.getMostLikelyWordChooser() .chooseMostLikelyWord(finalSequences, this.beamWidth); } } // have we holdover sequences? } // have we a most likely word chooser? if (!isHoldover) { for (LetterGuessObserver observer : observers) { observer.onBeamSearchEnd(bestSequence, finalSequences, holdoverSequences); } } } finally { MONITOR.endTask(); } MONITOR.startTask("assign letter"); try { if (shouldCombineWithHoldover) { holdoverSequences = null; } if (!isHoldover) { for (LetterGuessObserver observer : observers) { observer.onStartSequence(bestSequence); } group.setBestLetterSequence(bestSequence); int i = 0; for (ShapeInSequence shapeInSequence : bestSequence.getUnderlyingShapeSequence()) { String bestOutcome = bestSequence.get(i).getString(); this.assignLetter(shapeInSequence, bestOutcome); i++; } // next shape for (LetterGuessObserver observer : observers) { observer.onGuessSequence(bestSequence); } } this.shapeCount += group.getShapes().size(); if (this.currentMonitor != null) { double progress = (double) shapeCount / (double) totalShapeCount; LOG.debug("progress: " + progress); currentMonitor.setPercentComplete(progress); } } finally { MONITOR.endTask(); } } // next group } // next row } // next paragraph for (LetterGuessObserver observer : observers) { observer.onImageEnd(); } }
From source file:au.org.ala.layers.web.TabulationService.java
private double[] speciesTotals(List<Tabulation> tabulations, boolean row) throws IOException { //determine x & y field names TreeMap<String, String> objects1 = new TreeMap<String, String>(); TreeMap<String, String> objects2 = new TreeMap<String, String>(); for (Tabulation t : tabulations) { objects1.put(t.getPid1(), t.getName1()); objects2.put(t.getPid2(), t.getName2()); }//from w w w . j av a 2 s .c o m int rows = Math.max(objects1.size(), objects2.size()); int columns = Math.min(objects1.size(), objects2.size()); double[] grid = new double[row ? rows : columns]; //populate grid if (objects1.size() <= objects2.size()) { //row and column sort order and labels TreeMap<String, Integer> order1 = new TreeMap<String, Integer>(); TreeMap<String, Integer> order2 = new TreeMap<String, Integer>(); int pos = 0; for (String s : objects1.keySet()) { order1.put(s, pos++); } pos = 0; for (String s : objects2.keySet()) { order2.put(s, pos++); } //grid for (Tabulation t : tabulations) { //grid[order2.get(t.getPid2()) + 1][order1.get(t.getPid1()) + 1] = String.valueOf(t.getSpecies()); if (row) grid[order2.get(t.getPid2())] = t.getSpeciest2() == null ? 0 : t.getSpeciest2(); else grid[order1.get(t.getPid1())] = t.getSpeciest1() == null ? 0 : t.getSpeciest1(); } } else { //row and column sort order and labels TreeMap<String, Integer> order1 = new TreeMap<String, Integer>(); TreeMap<String, Integer> order2 = new TreeMap<String, Integer>(); int pos = 0; for (String s : objects1.keySet()) { order1.put(s, pos++); } pos = 0; for (String s : objects2.keySet()) { order2.put(s, pos++); } //grid for (Tabulation t : tabulations) { //grid[order1.get(t.getPid1()) + 1][order2.get(t.getPid2()) + 1] = String.valueOf(t.getSpecies()); if (!row) grid[order2.get(t.getPid2())] = t.getSpeciest2() == null ? 0 : t.getSpeciest2(); else grid[order1.get(t.getPid1())] = t.getSpeciest1() == null ? 0 : t.getSpeciest1(); } } return grid; }
From source file:nars.predict.RNNBeliefPrediction.java
@Override protected void train() { ///*from w ww . j av a 2 s. co m*/ //double[] target = {((data[x(i1)] + data[x(i2)])/2.0)}; //new Sample(data, target, 2, length, 1, 1); TreeMap<Integer, double[]> d = new TreeMap(); int cc = 0; int hd = Math.round(predictionTimeSpanFactor * nar.memory.getDuration() / 2f / downSample); for (Concept c : concepts) { for (Sentence s : c.beliefs) { if (s.isEternal()) { continue; } int o = (int) Math.round(((double) s.getOccurenceTime()) / ((double) downSample)); if (o > nar.time()) { continue; //non-future beliefs } for (int oc = o - hd; oc <= o + hd; oc++) { double[] x = d.get(oc); if (x == null) { x = new double[inputSize]; d.put(oc, x); } float freq = 2f * (s.truth.getFrequency() - 0.5f); float conf = s.truth.getConfidence(); if (freq < 0) { } x[cc] += freq * conf; } } cc++; } if (d.size() < 2) { data = null; return; } data = new SampleSet(); int first = d.firstKey(); int last = (int) nar.time(); if (last - first > maxDataFrames * downSample) { first = last - maxDataFrames * downSample; } int frames = (int) (last - first); int bsize = getInputSize() * frames; int isize = getPredictionSize() * frames; if (actual == null || actual.length != bsize) actual = new double[bsize]; else Arrays.fill(actual, 0); if (ideal == null || ideal.length != isize) ideal = new double[isize]; else Arrays.fill(ideal, 0); int idealSize = getPredictionSize(); int ac = 0, id = 0; double[] prevX = null; for (int i = first; i <= last; i++) { double[] x = d.get(i); if (x == null) { x = new double[inputSize]; } else { if (normalizeInputVectors) { x = normalize(x); } } if (prevX != null) { System.arraycopy(prevX, 0, actual, ac, inputSize); ac += inputSize; System.arraycopy(getTrainedPrediction(x), 0, ideal, id, idealSize); id += idealSize; } prevX = x; } Sample s = new Sample(actual, ideal, inputSize, idealSize); data.add(s); //System.out.println(data); if (trainer == null) { trainer = new GradientDescent(); trainer.setNet(net); trainer.setRnd(rnd); trainer.setPermute(true); trainer.setTrainingSet(data); trainer.setLearningRate(learningrate); trainer.setMomentum(momentum); trainer.setEpochs(trainIterationsPerCycle); trainer.setEarlyStopping(false); trainer.setOnline(true); trainer.setTargetError(0); trainer.clearListener(); } else { //trainer.reset(); } trainer.train(); //System.out.println("LSTM error: " + trainer.getTrainingError()); }