List of usage examples for java.util TreeMap descendingMap
NavigableMap descendingMap
To view the source code for java.util TreeMap descendingMap.
Click Source Link
From source file:org.polymap.kaps.ui.form.WohnungGrunddatenFormEditorPage.java
@SuppressWarnings("unchecked") @Override/*from w ww . j a va2 s .c o m*/ public void createFormContent(final IFormEditorPageSite site) { super.createFormContent(site); Composite newLine, lastLine = null; Composite parent = pageSite.getPageBody(); newLine = newFormField(IFormFieldLabel.NO_LABEL).setToolTipText("Objektnummer") .setProperty(new PropertyAdapter(wohnung.objektNummer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new NotNullMyNumberValidator(Integer.class)) .setEnabled(wohnung.objektNummer().get() == null).setLayoutData(left().left(0).right(15).create()) .create(); lastLine = newLine; Composite gebaeudeNummer = newFormField(IFormFieldLabel.NO_LABEL).setToolTipText("Gebudenummer") .setProperty(new PropertyAdapter(wohnung.gebaeudeNummer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new NotNullMyNumberValidator(Integer.class)) .setEnabled(wohnung.objektNummer().get() == null).setLayoutData(left().left(16).right(31).create()) .create(); final ActionButton openGebaeude = new ActionButton(parent, new Action("Gebude bearbeiten") { @Override public void run() { GebaeudeComposite gebaeude = GebaeudeComposite.Mixin.forKeys(wohnung.objektNummer().get(), wohnung.gebaeudeNummer().get()); if (gebaeude != null) { KapsPlugin.openEditor(fs, GebaeudeComposite.NAME, gebaeude); } } }); openGebaeude.setLayoutData(left().left(32).right(63).height(25).create()); openGebaeude.setEnabled(wohnung.objektNummer().get() != null && wohnung.objektNummer().get() != null); lastLine = newLine; newLine = newFormField(IFormFieldLabel.NO_LABEL).setToolTipText("Wohnungsnummer") .setProperty(new PropertyAdapter(wohnung.wohnungsNummer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new NotNullMyNumberValidator(Integer.class)) // .setEnabled( wohnung.wohnungsNummer().get() == null ) .setLayoutData(left().left(64).right(79).create()).create(); newFormField(IFormFieldLabel.NO_LABEL).setToolTipText("Fortfhrung") .setProperty(new PropertyAdapter(wohnung.wohnungsFortfuehrung())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new NotNullMyNumberValidator(Integer.class)) .setEnabled(wohnung.wohnungsNummer().get() == null) .setLayoutData(left().left(80).right(95).create()).create(); // flurstcke lastLine = newLine; TreeMap<String, FlurstueckComposite> flurstuecke = new TreeMap<String, FlurstueckComposite>(); if (wohnung.gebaeudeNummer().get() != null) { // its not a new wohnung without a gebaeude Iterable<FlurstueckComposite> iterable = WohnungComposite.Mixin.findFlurstueckeFor(wohnung); for (FlurstueckComposite flurstueck : iterable) { flurstuecke.put(flurstueck.name().get(), flurstueck); } } if (wohnung.flurstueck().get() != null) { flurstuecke.put(wohnung.flurstueck().get().name().get(), wohnung.flurstueck().get()); } newLine = newFormField("Lage") .setProperty(new AssociationAdapter<FlurstueckComposite>(wohnung.flurstueck())) .setField(new PicklistFormField(flurstuecke.descendingMap())) .setLayoutData(left().top(lastLine).right(RIGHT).create()).setParent(parent).create(); lastLine = newLine; newLine = newFormField("Eigentumsnr.").setToolTipText("Wohnungseigentumsnummer") .setProperty(new PropertyAdapter(wohnung.wohnungseigentumsNummer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setLayoutData(left().top(lastLine).create()).setParent(parent).create(); lastLine = newLine; newLine = newFormField("Eigentum").setToolTipText("Eigentum am Grundstck") .setProperty(new AssociationAdapter<EigentumsartComposite>(wohnung.eigentumsArt())) .setField(namedAssocationsPicklist(EigentumsartComposite.class)) .setLayoutData(left().top(lastLine).create()).setParent(parent).create(); newFormField("Anzahl Zimmer").setProperty(new PropertyAdapter(wohnung.anzahlZimmer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class)) .setLayoutData(right().right(75).top(lastLine).create()).setParent(parent).create(); lastLine = newLine; newLine = createFlaecheField("Wohn-/Nutzflche", "Woh-/Nutzflche in m", wohnung.wohnflaeche(), left().top(lastLine), parent, true); lastLine = newLine; newLine = newFormField("Gesamtnutzungsdauer").setToolTipText("Gesamtnutzungsdauer in Jahren") .setProperty(new PropertyAdapter(wohnung.gesamtNutzungsDauer())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class)).setLayoutData(left().top(lastLine).create()) .setParent(parent).create(); lastLine = newLine; newLine = newFormField("Baujahr tatschlich").setToolTipText("Baujahr tatschlich") .setProperty(new PropertyAdapter(wohnung.baujahr())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class)).setLayoutData(left().top(lastLine).create()) .setParent(parent).create(); Composite baujahrField = newFormField("Baujahr bereinigt").setToolTipText("Baujahr bereinigt") .setProperty(new PropertyAdapter(wohnung.bereinigtesBaujahr())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class)) .setLayoutData(right().right(75).top(lastLine).create()).setParent(parent).create(); site.addFieldListener( gndbjListener = new NonFiringFieldListener(wohnung.gesamtNutzungsDauer(), wohnung.baujahr())); // berechnen knopf baujahrBerechneAction = new ActionButton(parent, new Action("Berechnen") { @Override public void run() { if (gndbjListener.get(wohnung.gesamtNutzungsDauer()) == null || gndbjListener.get(wohnung.baujahr()) == null) { MessageDialog.openError(PolymapWorkbench.getShellToParentOn(), "Fehlende Daten", "Bitte geben Sie Gesamtnutzungsdauer und das tatschliche Baujahr ein, bevor Sie diese Berechnung starten."); } if (wohnung.vertrag().get() == null) { MessageDialog.openError(PolymapWorkbench.getShellToParentOn(), "Fehlende Daten", "Zu dieser Wohnung wurde kein Vertrag gefunden."); } else { ErmittlungModernisierungsgradComposite ermittlung = ErmittlungModernisierungsgradComposite.Mixin .forWohnung(wohnung); if (ermittlung == null) { ermittlung = repository.newEntity(ErmittlungModernisierungsgradComposite.class, null); ermittlung.wohnung().set(wohnung); ermittlung.vertrag().set(wohnung.vertrag().get()); ermittlung.objektNummer().set(wohnung.objektNummer().get()); ermittlung.gebaeudeNummer().set(wohnung.gebaeudeNummer().get()); ermittlung.wohnungsNummer().set(wohnung.wohnungsNummer().get()); ermittlung.wohnungsFortfuehrung().set(wohnung.wohnungsFortfuehrung().get()); ermittlung.alterObergrenzeZeile1().set(20.0d); ermittlung.alterObergrenzeZeile2().set(20.0d); ermittlung.alterObergrenzeZeile3().set(20.0d); ermittlung.alterObergrenzeZeile4().set(20.0d); ermittlung.alterObergrenzeZeile5().set(20.0d); ermittlung.alterObergrenzeZeile6().set(20.0d); ermittlung.alterObergrenzeZeile7().set(20.0d); ermittlung.alterObergrenzeZeile8().set(20.0d); } // ermittlung.gesamtNutzungsDauer().set( gndbjListener.get( // wohnung.gesamtNutzungsDauer() ) ); // ermittlung.tatsaechlichesBaujahr().set( gndbjListener.get( // wohnung.baujahr() ) ); FormEditor targetEditor = KapsPlugin.openEditor(fs, ErmittlungModernisierungsgradComposite.NAME, ermittlung); EventManager.instance() .publish(new InterEditorPropertyChangeEvent(formEditor, targetEditor, ermittlung, ermittlung.gesamtNutzungsDauer().qualifiedName().name(), ermittlung.gesamtNutzungsDauer().get(), gndbjListener.get(wohnung.gesamtNutzungsDauer()))); EventManager.instance().publish(new InterEditorPropertyChangeEvent(formEditor, targetEditor, ermittlung, ermittlung.tatsaechlichesBaujahr().qualifiedName().name(), ermittlung.tatsaechlichesBaujahr().get(), gndbjListener.get(wohnung.baujahr()))); } } }); baujahrBerechneAction.setLayoutData(right().left(baujahrField).top(lastLine).height(25).create()); baujahrBerechneAction.setEnabled(true); lastLine = newLine; newLine = newFormField("Etage").setProperty(new AssociationAdapter<EtageComposite>(wohnung.etage())) .setField(namedAssocationsPicklist(EtageComposite.class)) .setLayoutData(left().top(lastLine).create()).setParent(parent).create(); newFormField("Bemerkung").setToolTipText("Bemerkung zur Etage") .setProperty(new PropertyAdapter(wohnung.etageBeschreibung())).setField(new StringFormField()) .setLayoutData(right().top(lastLine).create()).setParent(parent).create(); lastLine = newLine; newLine = newFormField("Stockwerk").setToolTipText("Stockwerk entsprechend Statistischem Bundesamt") .setProperty(new AssociationAdapter<StockwerkStaBuComposite>(wohnung.stockwerkStaBu())) .setField(namedAssocationsPicklist(StockwerkStaBuComposite.class)) .setLayoutData(left().top(lastLine).create()).setParent(parent).create(); newLine = newFormField("Art").setToolTipText("Art entsprechend Statistischem Bundesamt") .setProperty(new AssociationAdapter<ImmobilienArtStaBuComposite>(wohnung.immobilienArtStaBu())) .setField(namedAssocationsPicklist(ImmobilienArtStaBuComposite.class)) .setLayoutData(right().top(lastLine).create()).setParent(parent).create(); lastLine = newLine; newLine = newFormField("Himmelsrichtung") .setProperty(new AssociationAdapter<HimmelsrichtungComposite>(wohnung.himmelsrichtung())) .setField(namedAssocationsPicklist(HimmelsrichtungComposite.class)) .setLayoutData(left().top(lastLine).create()).setParent(parent).create(); newFormField("Umbau").setToolTipText("Jahr des letzen Umbaus") .setProperty(new PropertyAdapter(wohnung.umbau())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Integer.class)).setLayoutData(right().top(lastLine).create()) .setParent(parent).create(); lastLine = newLine; newLine = newFormField("Bemerkung").setProperty(new PropertyAdapter(wohnung.bemerkung())) .setField(new TextFormField()).setLayoutData(left().right(RIGHT).height(50).top(lastLine).create()) .setParent(parent).create(); Section section = newSection(newLine, "Ausstattung"); Composite client = (Composite) section.getClient(); lastLine = newLine; newLine = newFormField("Balkon").setToolTipText("wertrelevanter Balkon?") .setProperty(new PropertyAdapter(wohnung.balkon())).setField(new BooleanFormField()) .setLayoutData(left().top(lastLine).create()).setParent(client).create(); newFormField("Terrasse").setToolTipText("wertrelevante Terrasse?") .setProperty(new PropertyAdapter(wohnung.terrasse())).setField(new BooleanFormField()) .setLayoutData(right().top(lastLine).create()).setParent(client).create(); lastLine = newLine; newLine = newFormField("Punkte").setToolTipText("Bewertungspunkte der Ausstattung") .setProperty(new PropertyAdapter(wohnung.bewertungsPunkte())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class)) .setLayoutData(left().right(40).top(lastLine).bottom(100).create()).setParent(client).create(); Composite schluessel = newFormField("Schlssel") .setProperty(new AssociationAdapter<AusstattungComposite>(wohnung.ausstattungSchluessel())) .setField(namedAssocationsPicklist(AusstattungComposite.class)) .setLayoutData(right().left(newLine).right(75).top(lastLine).create()).setParent(client).create(); site.addFieldListener(ausstattungListener = new IFormFieldListener() { @Override public void fieldChange(FormFieldEvent ev) { if (ev.getEventCode() == IFormFieldListener.VALUE_CHANGE && ev.getFieldName().equals(wohnung.bewertungsPunkte().qualifiedName().name())) { site.setFieldValue(wohnung.ausstattungSchluessel().qualifiedName().name(), AusstattungComposite.Mixin.forWert((Double) ev.getNewValue())); } } }); // berechnen knopf bewertungAction = new ActionButton(client, new Action("Ermitteln") { @Override public void run() { AusstattungBewertungComposite aust = AusstattungBewertungComposite.Mixin.forWohnung(wohnung); if (aust == null) { aust = repository.newEntity(AusstattungBewertungComposite.class, null); aust.wohnung().set(wohnung); } FormEditor targetEditor = KapsPlugin.openEditor(fs, AusstattungBewertungComposite.NAME, aust); } }); bewertungAction.setLayoutData(right().left(schluessel).top(lastLine).height(25).create()); bewertungAction.setEnabled(true); }
From source file:com.sg.capstone.dao.BlogDaoDbImpl.java
@Override public Map<Integer, List<String>> getAllUniqueYearMonths() { // create map that will have year as key and months as values TreeMap<Integer, List<String>> yearMonthsMap = new TreeMap<>(); // get the list of unique blogPost years to be used as reference for sorting months List<Integer> yearList = getAllUniqueYears(); // get SQL Year-Month results in form of year:month (ie. "2016:November") List<String> uniqueYearMonths_SqlResult = jdbcTemplate.queryForList(SQL_SELECT_ALL_UNIQUE_YEAR_MONTHS, String.class); // for loop to separate year:month string and then store in hashMap based on year for (int year : yearList) { List<String> monthList = new ArrayList<>(); for (String entry : uniqueYearMonths_SqlResult) { // split each year:month String to (ie.) ["2016", "November"] String[] currentYearMonthArray = entry.split(":"); Integer currentYear = Integer.parseInt(currentYearMonthArray[0]); String currentMonth = currentYearMonthArray[1]; if (currentYear == year) { monthList.add(currentMonth); }//from w w w. ja v a 2 s .c om } yearMonthsMap.put(year, monthList); } return yearMonthsMap.descendingMap(); }
From source file:ark.model.SupervisedModelCreg.java
@Override protected AssignmentList toParseInternalHelper(AssignmentList internalAssignments) { TreeMap<Double, List<String>> sortedWeights = new TreeMap<Double, List<String>>(); File modelFile = new File(this.modelPath.getValue()); if (!modelFile.exists()) return internalAssignments; try {/*w w w . j a v a 2 s . co m*/ BufferedReader br = new BufferedReader(new FileReader(modelFile)); String line = null; while ((line = br.readLine()) != null) { String[] lineParts = line.split("\t"); Double value = null; if (lineParts.length < 3) continue; try { value = Math.abs(Double.parseDouble(lineParts[2])); } catch (NumberFormatException e) { continue; } if (!sortedWeights.containsKey(value)) sortedWeights.put(value, new ArrayList<String>()); sortedWeights.get(value).add(line); } br.close(); } catch (Exception e) { e.printStackTrace(); } NavigableMap<Double, List<String>> descendingWeights = sortedWeights.descendingMap(); int i = 0; for (List<String> lines : descendingWeights.values()) { for (String line : lines) { internalAssignments .add(Assignment.assignmentTyped(null, Context.ARRAY_STR, "w-" + i, Obj.stringValue(line))); i++; } } return internalAssignments; }
From source file:com.opengamma.bloombergexample.loader.DemoEquityOptionCollarPortfolioLoader.java
protected ManageablePortfolio generatePortfolio(String portfolioName) { ReferenceDataProvider referenceDataProvider = ((BloombergToolContext) getToolContext()) .getBloombergReferenceDataProvider(); ManageablePortfolio portfolio = new ManageablePortfolio(portfolioName); //Is this a hack? ManageablePortfolioNode rootNode = portfolio.getRootNode(); portfolio.setRootNode(rootNode);/*from w w w . java 2 s . co m*/ // String indexTickerSuffix = " Index"; Set<String> memberEquities = new HashSet<String>(); for (Entry<String, String> entry : INDEXES_TO_EXCHANGE.entrySet()) { String indexTickerSuffix = " Index"; String underlying = entry.getKey(); String ticker = underlying + indexTickerSuffix; //don't add index (delete at some point) // addNodes(rootNode, ticker, false, INDEX_OPTION_PERIODS); Set<String> indexMembers = BloombergDataUtils.getIndexMembers(referenceDataProvider, ticker); for (String member : indexMembers) { String symbol = getBloombergEquitySymbol(entry.getValue(), member); //time series errors for Walmart //Todo: investegate & fix if ("WMT US Equity".equals(symbol)) { continue; } memberEquities.add(symbol); } } // Sort the symbols for the current index by market cap (highest to lowest), skipping any in the list of EXCLUDED_SECTORS TreeMap<Double, String> equityByMarketCap = new TreeMap<Double, String>(); ReferenceDataResult fields = referenceDataProvider.getFields(memberEquities, Sets .newHashSet(BloombergFields.CURRENT_MARKET_CAP_FIELD, BloombergConstants.FIELD_GICS_SUB_INDUSTRY)); for (String equity : memberEquities) { PerSecurityReferenceDataResult result = fields.getResult(equity); String gicsCodeString = result.getFieldData().getString(BloombergConstants.FIELD_GICS_SUB_INDUSTRY); GICSCode gicsCode = GICSCode.of(gicsCodeString); if (EXCLUDED_SECTORS.contains(gicsCode.getSectorDescription())) { continue; } Double marketCap = result.getFieldData().getDouble(BloombergFields.CURRENT_MARKET_CAP_FIELD); equityByMarketCap.put(marketCap, equity); } // Add a given number of symbols (MEMBERS_DEPTH) to the portfolio and store in a List // When adding to the portfolio, add a collar of options with PVs distributed equally +/- around 0 int count = 0; List<String> chosenEquities = new ArrayList<String>(); for (Entry<Double, String> entry : equityByMarketCap.descendingMap().entrySet()) { if (count++ >= _numMembers) { break; } addNodes(rootNode, entry.getValue(), true, MEMBER_OPTION_PERIODS); chosenEquities.add(entry.getValue()); } s_logger.info("Generated collar portfolio for {}", chosenEquities); return portfolio; }
From source file:org.apache.bookkeeper.metastore.TestMetaStore.java
/** * Test usage of (scan) operation on (full and partial) fields. *///from www . j a va2 s. c om @Test(timeout = 60000) public void testOpenCursor() throws Exception { TreeMap<String, Value> allValues = Maps.newTreeMap(); TreeMap<String, Value> partialValues = Maps.newTreeMap(); TreeMap<String, Value> nonValues = Maps.newTreeMap(); Set<String> counterFields = Sets.newHashSet(FIELD_COUNTER); for (int i = 5; i < 24; i++) { char c = (char) ('a' + i); String key = String.valueOf(c); Value v = makeValue("value" + i, i); Value cv = v.project(counterFields); Value nv = v.project(NON_FIELDS); myTable.put(key, new Value(v), Version.NEW); allValues.put(key, v); partialValues.put(key, cv); nonValues.put(key, nv); } // test open cursor MetastoreCursor cursor = myTable.openCursor(ALL_FIELDS); openCursorTest(cursor, allValues, 7); cursor = myTable.openCursor(counterFields); openCursorTest(cursor, partialValues, 7); cursor = myTable.openCursor(NON_FIELDS); openCursorTest(cursor, nonValues, 7); // test order inclusive exclusive Iterator<Map.Entry<String, Value>> expectedIterator; expectedIterator = allValues.subMap("l", true, "u", true).entrySet().iterator(); openRangeCursorTest("l", true, "u", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.descendingMap().subMap("u", true, "l", true).entrySet().iterator(); openRangeCursorTest("u", true, "l", true, Order.DESC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.subMap("l", false, "u", false).entrySet().iterator(); openRangeCursorTest("l", false, "u", false, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.descendingMap().subMap("u", false, "l", false).entrySet().iterator(); openRangeCursorTest("u", false, "l", false, Order.DESC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.subMap("l", true, "u", false).entrySet().iterator(); openRangeCursorTest("l", true, "u", false, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.descendingMap().subMap("u", true, "l", false).entrySet().iterator(); openRangeCursorTest("u", true, "l", false, Order.DESC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.subMap("l", false, "u", true).entrySet().iterator(); openRangeCursorTest("l", false, "u", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.descendingMap().subMap("u", false, "l", true).entrySet().iterator(); openRangeCursorTest("u", false, "l", true, Order.DESC, ALL_FIELDS, expectedIterator, 7); // test out of range String firstKey = "f"; String lastKey = "x"; expectedIterator = allValues.subMap(firstKey, true, lastKey, true).entrySet().iterator(); openRangeCursorTest("a", true, "z", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.subMap("l", true, lastKey, true).entrySet().iterator(); openRangeCursorTest("l", true, "z", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.subMap(firstKey, true, "u", true).entrySet().iterator(); openRangeCursorTest("a", true, "u", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); // test EMPTY_START_KEY and EMPTY_END_KEY expectedIterator = allValues.subMap(firstKey, true, "u", true).entrySet().iterator(); openRangeCursorTest(EMPTY_START_KEY, true, "u", true, Order.ASC, ALL_FIELDS, expectedIterator, 7); expectedIterator = allValues.descendingMap().subMap(lastKey, true, "l", true).entrySet().iterator(); openRangeCursorTest(EMPTY_END_KEY, true, "l", true, Order.DESC, ALL_FIELDS, expectedIterator, 7); // test illegal arguments try { myTable.openCursor("a", true, "z", true, Order.DESC, ALL_FIELDS); fail("Should fail with wrong range"); } catch (MSException.IllegalOpException ioe) { } try { myTable.openCursor("z", true, "a", true, Order.ASC, ALL_FIELDS); fail("Should fail with wrong range"); } catch (MSException.IllegalOpException ioe) { } try { myTable.openCursor("a", true, "z", true, null, ALL_FIELDS); fail("Should fail with null order"); } catch (MSException.IllegalOpException ioe) { } }
From source file:com.opengamma.examples.bloomberg.loader.DemoEquityOptionCollarPortfolioLoader.java
protected ManageablePortfolio generatePortfolio(final String portfolioName) { final ReferenceDataProvider referenceDataProvider = getToolContext().getBloombergReferenceDataProvider(); final ManageablePortfolio portfolio = new ManageablePortfolio(portfolioName); //Is this a hack? final ManageablePortfolioNode rootNode = portfolio.getRootNode(); portfolio.setRootNode(rootNode);// w w w. j ava2 s . c o m // String indexTickerSuffix = " Index"; final Set<String> memberEquities = new HashSet<String>(); for (final Entry<String, String> entry : INDEXES_TO_EXCHANGE.entrySet()) { final String indexTickerSuffix = " Index"; final String underlying = entry.getKey(); final String ticker = underlying + indexTickerSuffix; //don't add index (delete at some point) // addNodes(rootNode, ticker, false, INDEX_OPTION_PERIODS); final Set<String> indexMembers = BloombergDataUtils.getIndexMembers(referenceDataProvider, ticker); for (final String member : indexMembers) { final String symbol = getBloombergEquitySymbol(entry.getValue(), member); //time series errors for Walmart //Todo: investegate & fix if ("WMT US Equity".equals(symbol)) { continue; } memberEquities.add(symbol); } } // Sort the symbols for the current index by market cap (highest to lowest), skipping any in the list of EXCLUDED_SECTORS final TreeMap<Double, String> equityByMarketCap = new TreeMap<Double, String>(); final Map<String, FudgeMsg> refDataMap = referenceDataProvider.getReferenceData(memberEquities, Sets .newHashSet(BloombergFields.CURRENT_MARKET_CAP_FIELD, BloombergConstants.FIELD_GICS_SUB_INDUSTRY)); for (final String equity : memberEquities) { final FudgeMsg fieldData = refDataMap.get(equity); if (fieldData == null) { throw new OpenGammaRuntimeException("Information not found for equity: " + equity); } final String gicsCodeString = fieldData.getString(BloombergConstants.FIELD_GICS_SUB_INDUSTRY); if (gicsCodeString == null) { continue; } final GICSCode gicsCode = GICSCode.of(gicsCodeString); if (EXCLUDED_SECTORS.contains(gicsCode.getSectorDescription())) { continue; } final Double marketCap = fieldData.getDouble(BloombergFields.CURRENT_MARKET_CAP_FIELD); if (marketCap != null) { equityByMarketCap.put(marketCap, equity); } } // Add a given number of symbols (MEMBERS_DEPTH) to the portfolio and store in a List // When adding to the portfolio, add a collar of options with PVs distributed equally +/- around 0 int count = 0; final List<String> chosenEquities = new ArrayList<String>(); for (final Entry<Double, String> entry : equityByMarketCap.descendingMap().entrySet()) { try { addNodes(rootNode, entry.getValue(), true, MEMBER_OPTION_PERIODS); chosenEquities.add(entry.getValue()); if (++count >= _numMembers) { break; } } catch (final RuntimeException e) { s_logger.warn("Caught exception", e); } } s_logger.info("Generated collar portfolio for {}", chosenEquities); return portfolio; }
From source file:org.polymap.kaps.ui.form.VertragsdatenBaulandGrunddatenFormEditorPage.java
@SuppressWarnings("unchecked") @Override// w w w . j a va2 s . c o m public void createFormContent(final IFormEditorPageSite site) { super.createFormContent(site); Composite newLine, lastLine = null; Composite parent = pageSite.getPageBody(); final VertragComposite kaufvertrag = vb.vertrag().get(); String nummer = EingangsNummerFormatter.format(kaufvertrag.eingangsNr().get()); String label = kaufvertrag == null ? "Kein Vertrag zugewiesen" : "Vertrag " + nummer + " ffnen"; ActionButton openVertrag = new ActionButton(parent, new Action(label) { @Override public void run() { KapsPlugin.openEditor(fs, VertragComposite.NAME, kaufvertrag); } }); openVertrag.setLayoutData(left().height(25).create()); openVertrag.setEnabled(kaufvertrag != null); Section section = newSection(parent, "Richtwertberechnung"); section.setLayoutData(new SimpleFormData(SECTION_SPACING).left(0).right(100).top(openVertrag).create()); Composite client = (Composite) section.getClient(); newLine = newFormField("Lageklasse").setProperty(new PropertyAdapter(vb.lageklasse())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().right(33).top(lastLine).create()).setParent(client).create(); newFormField("zul. GFZ").setProperty(new PropertyAdapter(vb.gfz())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(33).right(66).top(lastLine).create()).setParent(client).create(); newFormField("zul. Vollgeschosse").setProperty(new PropertyAdapter(vb.zulaessigeVollgeschosse())) .setField(new StringFormField()).setLayoutData(right().left(66).top(lastLine).create()) .setParent(client).create(); // lastLine = newLine; // newLine = newFormField( "Gebudeart" ).setEnabled( false ) // .setProperty( new AssociationAdapter<GebaeudeArtComposite>( // vb.gebaeudeArt() ) ) // .setField( namedAssocationsPicklist( GebaeudeArtComposite.class ) ) // .setLayoutData( left().top( lastLine ).create() ).setParent( client // ).create(); lastLine = newLine; newLine = newFormField("Bodennutzung") .setProperty(new AssociationAdapter<BodennutzungComposite>(vb.bodennutzung())) .setField(namedAssocationsPicklist(BodennutzungComposite.class)) .setLayoutData(left().top(lastLine).create()).setParent(client).create(); newFormField("Erschlieungsbeitrag") .setProperty(new AssociationAdapter<ErschliessungsBeitragComposite>(vb.erschliessungsBeitrag())) .setField(namedAssocationsPicklist(ErschliessungsBeitragComposite.class)) .setLayoutData(right().top(lastLine).create()).setParent(client).create(); lastLine = newLine; newLine = newFormField("Baujahr tatschlich").setProperty(new PropertyAdapter(vb.baujahr())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Integer.class)).setLayoutData(left().top(lastLine).create()) .setParent(client).create(); newFormField("Baujahr bereinigt").setProperty(new PropertyAdapter(vb.baujahrBereinigt())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Integer.class)).setLayoutData(right().top(lastLine).create()) .setParent(client).create(); lastLine = newLine; TreeMap<String, Object> zonen = new TreeMap<String, Object>(); TreeMap<String, Object> zeitraeume = new TreeMap<String, Object>(); // FlurstueckComposite flurstueck = vb.flurstueck().get(); RichtwertzoneComposite richtwertZone = vb.vertrag().get().richtwertZoneBauland().get(); if (richtwertZone != null) { String prefix = richtwertZone.schl().get(); zonen.put(prefix + " - " + richtwertZone.name().get(), richtwertZone); for (RichtwertzoneZeitraumComposite zeitraum : RichtwertzoneZeitraumComposite.Mixin .forZone(richtwertZone)) { if (zeitraum.gueltigAb().get() != null) { zeitraeume.put(KapsRepository.SHORT_DATE.format(zeitraum.gueltigAb().get()), zeitraum); } else { throw new RuntimeException( "Gltig ab fehlt in Richtwertzone: " + richtwertZone.schl().get() + " "); } } } //log.error(vb.richtwertZone().get().schl().get() + " is set"); newLine = newFormField("Richtwertzone").setEnabled(true) .setProperty(new AssociationAdapter<RichtwertzoneComposite>(vb.richtwertZone())) .setField(new PicklistFormField(zonen)).setLayoutData(left().top(lastLine).bottom(100).create()) .setParent(client).create(); newFormField("Gltig ab") .setProperty(new AssociationAdapter<RichtwertzoneZeitraumComposite>(vb.richtwertZoneG())) .setField(new PicklistFormField(zeitraeume.descendingMap())).setEnabled(true) .setLayoutData(right().top(lastLine).create()).setParent(client).create(); pageSite.addFieldListener(richtwertzone = new IFormFieldListener() { @Override public void fieldChange(FormFieldEvent ev) { if (ev.getEventCode() == IFormFieldListener.VALUE_CHANGE && ev.getFieldName().equals(vb.richtwertZoneG().qualifiedName().name())) { RichtwertzoneZeitraumComposite rzc = (RichtwertzoneZeitraumComposite) ev.getNewValue(); pageSite.setFieldValue(vb.richtwert().qualifiedName().name(), rzc != null && rzc.euroQm().get() != null ? NumberFormatter.getFormatter(2).format(rzc.euroQm().get()) : "0"); pageSite.setFieldValue(vb.erschliessungsBeitrag().qualifiedName().name(), rzc != null ? rzc.erschliessungsBeitrag().get() : null); } } }); section = newSection(section, "Bodenpreisberechnung"); client = (Composite) section.getClient(); lastLine = newLine; newLine = newFormField("Richtwert").setToolTipText("Richtwert aus Richtwertzone").setEnabled(false) .setProperty(new PropertyAdapter(vb.richtwert())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); lastLine = newLine; newLine = newFormField("GFZ-bereinigt").setToolTipText("GFZ bereinigter Bodenpreis").setEnabled(false) .setProperty(new PropertyAdapter(vb.gfzBereinigterBodenpreis())) .setValidator(new MyNumberValidator(Double.class, 2)) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); newFormField("verwenden?").setToolTipText("GFZ bereinigten Bodenpreis verwenden?") .setProperty(new PropertyAdapter(vb.gfzBereinigtenBodenpreisVerwenden())) .setField(new CheckboxFormField()).setLayoutData(right().right(75).top(lastLine).create()) .setParent(client).create(); newFormField("GFZ-Bereich").setEnabled(false).setProperty(new PropertyAdapter(vb.gfzBereich())) .setField(new StringFormField()).setLayoutData(right().left(75).top(lastLine).create()) .setParent(client).create(); lastLine = newLine; newLine = newFormField("Bereinigung in /m").setToolTipText("Richtwertbereinigung in /m (+/-)") .setProperty(new PropertyAdapter(vb.richtwertBereinigung())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); newFormField("Bemerkung").setToolTipText("Bemerkung zur Richwertbereinigung") .setProperty(new PropertyAdapter(vb.richtwertBereinigungBemerkung())) .setField(new StringFormField()).setLayoutData(right().top(lastLine).create()).setParent(client) .create(); lastLine = newLine; newLine = newFormField("Zuschlag in %").setToolTipText("Richtwertzuschlag in %") .setProperty(new PropertyAdapter(vb.richtwertZuschlagProzent())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().right(25).top(lastLine).create()).setParent(client).create(); newLine = newFormField("in /m").setEnabled(false) .setProperty(new PropertyAdapter(vb.richtwertZuschlagBerechnet())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); site.addFieldListener(riwezuschlag = new FieldCalculation(site, 2, vb.richtwertZuschlagBerechnet(), vb.richtwert(), vb.richtwertBereinigung(), vb.richtwertZuschlagProzent()) { @Override protected Double calculate(org.polymap.kaps.ui.FieldCalculation.ValueProvider values) { Double zuschlag = values.get(vb.richtwertZuschlagProzent()); if (zuschlag != null) { Double richtwert = values.get(vb.richtwert()); if (richtwert != null) { Double richtwertB = values.get(vb.richtwertBereinigung()); if (richtwertB != null) { richtwert += richtwertB; } return richtwert * zuschlag / 100; } } return null; } }); newFormField("Bemerkung").setToolTipText("Bemerkung zum Richtwertzuschlag") .setProperty(new PropertyAdapter(vb.richtwertZuschlagBemerkung())).setField(new StringFormField()) .setLayoutData(right().top(lastLine).create()).setParent(client).create(); lastLine = newLine; newLine = newFormField("Abschlag in %").setToolTipText("Richtwertabschlag in %") .setProperty(new PropertyAdapter(vb.richtwertAbschlagProzent())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().right(25).top(lastLine).create()).setParent(client).create(); newLine = newFormField("in /m").setEnabled(false) .setProperty(new PropertyAdapter(vb.richtwertAbschlagBerechnet())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); pageSite.addFieldListener(riweabschlag = new FieldCalculation(site, 2, vb.richtwertAbschlagBerechnet(), vb.richtwert(), vb.richtwertBereinigung(), vb.richtwertAbschlagProzent()) { @Override protected Double calculate(org.polymap.kaps.ui.FieldCalculation.ValueProvider values) { Double abschlag = values.get(vb.richtwertAbschlagProzent()); if (abschlag != null) { Double richtwert = values.get(vb.richtwert()); if (richtwert != null) { Double richtwertB = values.get(vb.richtwertBereinigung()); if (richtwertB != null) { richtwert += richtwertB; } return richtwert * abschlag / 100; } } return null; } }); newFormField("Bemerkung").setToolTipText("Bemerkung zum Richtwertabschlag") .setProperty(new PropertyAdapter(vb.richtwertAbschlagBemerkung())).setField(new StringFormField()) .setLayoutData(right().top(lastLine).create()).setParent(client).create(); lastLine = newLine; newLine = newFormField("Erschlieung in /m") .setToolTipText("Erschlieungskosten in /m anrechenbarer Grundstcksgre") .setProperty(new PropertyAdapter(vb.erschliessungsKosten())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).create()).setParent(client).create(); newFormField("Bemerkung").setToolTipText("Bemerkung gesamt") .setProperty(new PropertyAdapter(vb.bemerkungen())).setField(new TextFormField()) .setLayoutData(right().top(lastLine).height(100).bottom(100).create()).setParent(client).create(); // lastLine = newLine; // newLine = newFormField( "Preis unbebaut in /m" ).setToolTipText( // "Bodenpreis unbebaut in /m" ) // .setEnabled( false ).setProperty( new PropertyAdapter( // vb.bodenpreisUnbebaut() ) ) // .setField( new StringFormField( StringFormField.Style.ALIGN_RIGHT ) ) // .setValidator( new MyNumberValidator( Double.class, 2 ) ) // .setLayoutData( left().left( 25 ).top( lastLine ).create() ).setParent( // client ).create(); // pageSite.addFieldListener( preisunbebaut = new FieldCalculation( pageSite, // 2, vb.bodenpreisUnbebaut(), vb // .richtwert(), vb.richtwertBereinigung(), vb.richtwertZuschlagBerechnet(), // vb // .richtwertAbschlagBerechnet(), vb.erschliessungsKosten() ) { // // @Override // protected Double calculate( // org.polymap.kaps.ui.FieldCalculation.ValueProvider values ) { // Double richtwert = values.get( vb.richtwert() ); // if (richtwert != null) { // Double richtwertB = values.get( vb.richtwertBereinigung() ); // if (richtwertB != null) { // richtwert += richtwertB; // } // richtwertB = values.get( vb.richtwertZuschlagBerechnet() ); // if (richtwertB != null) { // richtwert += richtwertB; // } // richtwertB = values.get( vb.richtwertAbschlagBerechnet() ); // if (richtwertB != null) { // richtwert -= richtwertB; // } // richtwertB = values.get( vb.erschliessungsKosten() ); // if (richtwertB != null) { // richtwert -= richtwertB; // } // return richtwert; // } // return null; // } // } ); // lastLine = newLine; // newLine = newFormField( "Beb.-abschlag in %" ).setProperty( new // PropertyAdapter( vb.bebAbschlag() ) ) // .setField( new StringFormField( StringFormField.Style.ALIGN_RIGHT ) ) // .setValidator( new MyNumberValidator( Double.class, 2 ) ) // .setLayoutData( left().right( 25 ).top( lastLine ).create() ).setParent( // client ).create(); // newLine = newFormField( "in /m" ).setEnabled( false ) // .setProperty( new PropertyAdapter( vb.bebAbschlagBerechnet() ) ) // .setField( new StringFormField( StringFormField.Style.ALIGN_RIGHT ) ) // .setValidator( new MyNumberValidator( Double.class, 2 ) ) // .setLayoutData( left().left( 25 ).top( lastLine ).create() ).setParent( // client ).create(); // pageSite.addFieldListener( bebabschlag = new FieldCalculation( pageSite, // 2, vb.bebAbschlagBerechnet(), vb // .bodenpreisBebaut(), vb.bebAbschlag() ) { // // @Override // protected Double calculate( // org.polymap.kaps.ui.FieldCalculation.ValueProvider values ) { // Double abschlag = values.get( vb.bebAbschlag() ); // if (abschlag != null) { // Double bodenpreis = values.get( vb.bodenpreisBebaut() ); // if (bodenpreis != null) { // return bodenpreis * abschlag / 100; // } // } // return null; // } // } ); lastLine = newLine; newLine = newFormField("Richtwert in /m").setEnabled(false) .setToolTipText("Bereinigter Richtwert/Bodenwert in /m") .setProperty(new PropertyAdapter(vb.bodenpreisBebaut())) .setField(new StringFormField(StringFormField.Style.ALIGN_RIGHT)) .setValidator(new MyNumberValidator(Double.class, 2)) .setLayoutData(left().left(25).top(lastLine).bottom(100).create()).setParent(client).create(); pageSite.addFieldListener(bodenpreisbebaut = new FieldCalculation(pageSite, 2, vb.bodenpreisBebaut(), vb.richtwert(), vb.richtwertBereinigung(), vb.richtwertZuschlagBerechnet(), vb.richtwertAbschlagBerechnet(), vb.erschliessungsKosten()) { @Override protected Double calculate(org.polymap.kaps.ui.FieldCalculation.ValueProvider values) { Double richtwert = values.get(vb.richtwert()); if (richtwert != null) { Double richtwertB = values.get(vb.richtwertBereinigung()); if (richtwertB != null) { richtwert += richtwertB; } richtwertB = values.get(vb.richtwertZuschlagBerechnet()); if (richtwertB != null) { richtwert += richtwertB; } richtwertB = values.get(vb.richtwertAbschlagBerechnet()); if (richtwertB != null) { richtwert -= richtwertB; } richtwertB = values.get(vb.erschliessungsKosten()); if (richtwertB != null) { richtwert -= richtwertB; } return richtwert; } return null; } }); section = createBewertungenForm(section); }
From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.java
/** * Get the configuration opProps for the given operator. * These can be operator specific settings or settings from matching templates. * @param pa/*from ww w .j a va 2 s. com*/ * @param opConfs * @param appName */ private Map<String, String> getProperties(PropertyArgs pa, List<OperatorConf> opConfs, String appName) { Map<String, String> opProps = Maps.newHashMap(); Map<String, TemplateConf> templates = stramConf.getChildren(StramElement.TEMPLATE); // list of all templates that match operator, ordered by priority if (!templates.isEmpty()) { TreeMap<Integer, TemplateConf> matchingTemplates = getMatchingTemplates(pa, appName, templates); if (matchingTemplates != null && !matchingTemplates.isEmpty()) { // combined map of prioritized template settings for (TemplateConf t : matchingTemplates.descendingMap().values()) { opProps.putAll(Maps.fromProperties(t.properties)); } } List<TemplateConf> refTemplates = getDirectTemplates(opConfs, templates); for (TemplateConf t : refTemplates) { opProps.putAll(Maps.fromProperties(t.properties)); } } // direct settings // Apply the configurations in reverse order since the higher priority ones are at the beginning for (int i = opConfs.size() - 1; i >= 0; i--) { Conf conf1 = opConfs.get(i); opProps.putAll(Maps.fromProperties(conf1.properties)); } return opProps; }
From source file:org.apache.hadoop.hbase.master.LoadBalancer.java
/** * Generate a global load balancing plan according to the specified map of * server information to the most loaded regions of each server. * * The load balancing invariant is that all servers are within 1 region of the * average number of regions per server. If the average is an integer number, * all servers will be balanced to the average. Otherwise, all servers will * have either floor(average) or ceiling(average) regions. * * The algorithm is currently implemented as such: * * <ol>//from ww w . j av a 2 s.c o m * <li>Determine the two valid numbers of regions each server should have, * <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average). * * <li>Iterate down the most loaded servers, shedding regions from each so * each server hosts exactly <b>MAX</b> regions. Stop once you reach a * server that already has <= <b>MAX</b> regions. * <p> * Order the regions to move from most recent to least. * * <li>Iterate down the least loaded servers, assigning regions so each server * has exactly </b>MIN</b> regions. Stop once you reach a server that * already has >= <b>MIN</b> regions. * * Regions being assigned to underloaded servers are those that were shed * in the previous step. It is possible that there were not enough * regions shed to fill each underloaded server to <b>MIN</b>. If so we * end up with a number of regions required to do so, <b>neededRegions</b>. * * It is also possible that we were able fill each underloaded but ended * up with regions that were unassigned from overloaded servers but that * still do not have assignment. * * If neither of these conditions hold (no regions needed to fill the * underloaded servers, no regions leftover from overloaded servers), * we are done and return. Otherwise we handle these cases below. * * <li>If <b>neededRegions</b> is non-zero (still have underloaded servers), * we iterate the most loaded servers again, shedding a single server from * each (this brings them from having <b>MAX</b> regions to having * <b>MIN</b> regions). * * <li>We now definitely have more regions that need assignment, either from * the previous step or from the original shedding from overloaded servers. * * Iterate the least loaded servers filling each to <b>MIN</b>. * * <li>If we still have more regions that need assignment, again iterate the * least loaded servers, this time giving each one (filling them to * </b>MAX</b>) until we run out. * * <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions. * * In addition, any server hosting >= <b>MAX</b> regions is guaranteed * to end up with <b>MAX</b> regions at the end of the balancing. This * ensures the minimal number of regions possible are moved. * </ol> * * TODO: We can at-most reassign the number of regions away from a particular * server to be how many they report as most loaded. * Should we just keep all assignment in memory? Any objections? * Does this mean we need HeapSize on HMaster? Or just careful monitor? * (current thinking is we will hold all assignments in memory) * * @param clusterState Map of regionservers and their load/region information to * a list of their most loaded regions * @return a list of regions to be moved, including source and destination, * or null if cluster is already balanced */ public List<RegionPlan> balanceCluster(Map<HServerInfo, List<HRegionInfo>> clusterState) { long startTime = System.currentTimeMillis(); // Make a map sorted by load and count regions TreeMap<HServerInfo, List<HRegionInfo>> serversByLoad = new TreeMap<HServerInfo, List<HRegionInfo>>( new HServerInfo.LoadComparator()); int numServers = clusterState.size(); if (numServers == 0) { LOG.debug("numServers=0 so skipping load balancing"); return null; } int numRegions = 0; // Iterate so we can count regions as we build the map for (Map.Entry<HServerInfo, List<HRegionInfo>> server : clusterState.entrySet()) { server.getKey().getLoad().setNumberOfRegions(server.getValue().size()); numRegions += server.getKey().getLoad().getNumberOfRegions(); serversByLoad.put(server.getKey(), server.getValue()); } // Check if we even need to do any load balancing float average = (float) numRegions / numServers; // for logging // HBASE-3681 check sloppiness first int floor = (int) Math.floor(average * (1 - slop)); int ceiling = (int) Math.ceil(average * (1 + slop)); if (serversByLoad.lastKey().getLoad().getNumberOfRegions() <= ceiling && serversByLoad.firstKey().getLoad().getNumberOfRegions() >= floor) { // Skipped because no server outside (min,max) range LOG.info("Skipping load balancing. servers=" + numServers + " " + "regions=" + numRegions + " average=" + average + " " + "mostloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions() + " leastloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions()); return null; } int min = numRegions / numServers; int max = numRegions % numServers == 0 ? min : min + 1; // Balance the cluster // TODO: Look at data block locality or a more complex load to do this List<RegionPlan> regionsToMove = new ArrayList<RegionPlan>(); int regionidx = 0; // track the index in above list for setting destination // Walk down most loaded, pruning each to the max int serversOverloaded = 0; Map<HServerInfo, BalanceInfo> serverBalanceInfo = new TreeMap<HServerInfo, BalanceInfo>(); for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.descendingMap().entrySet()) { HServerInfo serverInfo = server.getKey(); int regionCount = serverInfo.getLoad().getNumberOfRegions(); if (regionCount <= max) { serverBalanceInfo.put(serverInfo, new BalanceInfo(0, 0)); break; } serversOverloaded++; List<HRegionInfo> regions = randomize(server.getValue()); int numToOffload = Math.min(regionCount - max, regions.size()); int numTaken = 0; for (int i = regions.size() - 1; i >= 0; i--) { HRegionInfo hri = regions.get(i); // Don't rebalance meta regions. if (hri.isMetaRegion()) continue; regionsToMove.add(new RegionPlan(hri, serverInfo, null)); numTaken++; if (numTaken >= numToOffload) break; } serverBalanceInfo.put(serverInfo, new BalanceInfo(numToOffload, (-1) * numTaken)); } // Walk down least loaded, filling each to the min int serversUnderloaded = 0; // number of servers that get new regions int neededRegions = 0; // number of regions needed to bring all up to min for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= min) { break; } serversUnderloaded++; int numToTake = min - regionCount; int numTaken = 0; while (numTaken < numToTake && regionidx < regionsToMove.size()) { regionsToMove.get(regionidx).setDestination(server.getKey()); numTaken++; regionidx++; } serverBalanceInfo.put(server.getKey(), new BalanceInfo(0, numTaken)); // If we still want to take some, increment needed if (numTaken < numToTake) { neededRegions += (numToTake - numTaken); } } // If none needed to fill all to min and none left to drain all to max, // we are done if (neededRegions == 0 && regionidx == regionsToMove.size()) { long endTime = System.currentTimeMillis(); LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. " + "Moving " + regionsToMove.size() + " regions off of " + serversOverloaded + " overloaded servers onto " + serversUnderloaded + " less loaded servers"); return regionsToMove; } // Need to do a second pass. // Either more regions to assign out or servers that are still underloaded // If we need more to fill min, grab one from each most loaded until enough if (neededRegions != 0) { // Walk down most loaded, grabbing one from each until we get enough for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.descendingMap().entrySet()) { BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey()); int idx = balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload(); if (idx >= server.getValue().size()) break; HRegionInfo region = server.getValue().get(idx); if (region.isMetaRegion()) continue; // Don't move meta regions. regionsToMove.add(new RegionPlan(region, server.getKey(), null)); if (--neededRegions == 0) { // No more regions needed, done shedding break; } } } // Now we have a set of regions that must be all assigned out // Assign each underloaded up to the min, then if leftovers, assign to max // Walk down least loaded, assigning to each to fill up to min for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= min) break; BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey()); if (balanceInfo != null) { regionCount += balanceInfo.getNumRegionsAdded(); } if (regionCount >= min) { continue; } int numToTake = min - regionCount; int numTaken = 0; while (numTaken < numToTake && regionidx < regionsToMove.size()) { regionsToMove.get(regionidx).setDestination(server.getKey()); numTaken++; regionidx++; } } // If we still have regions to dish out, assign underloaded to max if (regionidx != regionsToMove.size()) { for (Map.Entry<HServerInfo, List<HRegionInfo>> server : serversByLoad.entrySet()) { int regionCount = server.getKey().getLoad().getNumberOfRegions(); if (regionCount >= max) { break; } regionsToMove.get(regionidx).setDestination(server.getKey()); regionidx++; if (regionidx == regionsToMove.size()) { break; } } } long endTime = System.currentTimeMillis(); if (regionidx != regionsToMove.size() || neededRegions != 0) { // Emit data so can diagnose how balancer went astray. LOG.warn("regionidx=" + regionidx + ", regionsToMove=" + regionsToMove.size() + ", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded + ", serversUnderloaded=" + serversUnderloaded); StringBuilder sb = new StringBuilder(); for (Map.Entry<HServerInfo, List<HRegionInfo>> e : clusterState.entrySet()) { if (sb.length() > 0) sb.append(", "); sb.append(e.getKey().getServerName()); sb.append(" "); sb.append(e.getValue().size()); } LOG.warn("Input " + sb.toString()); } // All done! LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. " + "Moving " + regionsToMove.size() + " regions off of " + serversOverloaded + " overloaded servers onto " + serversUnderloaded + " less loaded servers"); return regionsToMove; }
From source file:org.apache.hadoop.mapred.HFSPScheduler.java
/** * Preempt missingSlots number of slots from jobs bigger * // w w w. j a va 2s .c om * @param jip * Job that clamies slots * @param allJobs * all the size based jobs in the cluster * @param localJobs * size based jobs that can be immediately suspended * @param missingSlots * number of slots to claim * @param numToSkip * number of slots on non-local * * @return number of tasks preemped in the cluster for jip. The first elem of * the tuple is the number of tasks preempted for new tasks, the * second in the number of tasks preempted for tasks to be resumed */ private ClaimedSlots claimSlots(HelperForType helper, final Phase phase, final JobInProgress jip, int missingNewSlots, int missingResumableSlots, int numToSkip, TreeMap<JobDurationInfo, JobInProgress> allJobs, TreeMap<JobDurationInfo, TaskStatuses> localJobs) { assert phase == Phase.SIZE_BASED || missingResumableSlots == 0; final TaskType type = helper.taskType; JobDurationInfo jdi = this.getDuration(jip.getJobID(), type); /* #size based tasks that occupies train slots in the cluster (suspendable) */ int numTasksToPreempt = 0; if (phase == Phase.TRAIN) { /** num of size based tasks that can be suspended for training */ int numOverflowSizeBasedTasks = helper.maxSizeBasedSlots > helper.runningSizeBasedTasks ? 0 : helper.runningSizeBasedTasks - helper.maxSizeBasedSlots; /* num of size base tasks to preempt for the training of jip */ numTasksToPreempt = Math.min(missingNewSlots, numOverflowSizeBasedTasks); if (LOG.isDebugEnabled()) { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "):" + " numOverflowSizeBasedTasks: " + numOverflowSizeBasedTasks + " numTasksToPreempt: " + numTasksToPreempt + " missingNewSlots: " + missingNewSlots + " numTrainTasksForJob: " + helper.numTrainTasksForJob + " canAssignTrain: " + helper.canAssignTrain() + " numToSkip: " + numToSkip); } } else { numTasksToPreempt = missingNewSlots; if (LOG.isDebugEnabled()) { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "):" + " missingNewSlots: " + missingNewSlots + " missingResumableSlots: " + missingResumableSlots + " numTrainTasksForJob: " + helper.numTrainTasksForJob + " canAssignTrain: " + helper.canAssignTrain() + " numToSkip: " + numToSkip); } } final int startingNumTasksToPreemptForNew = numTasksToPreempt; final int startingResumableSlots = missingResumableSlots; // try to free pendingTasks number of slots among running on this TT Iterator<Entry<JobDurationInfo, JobInProgress>> sizeBasedJobsDescIter = allJobs.descendingMap().entrySet() .iterator(); Iterator<Entry<JobDurationInfo, TaskStatuses>> sizeBasedJobsDescIterOnTT = localJobs.entrySet().iterator(); Entry<JobDurationInfo, TaskStatuses> biggerOnTT = sizeBasedJobsDescIterOnTT.hasNext() ? sizeBasedJobsDescIterOnTT.next() : null; while (this.preemptionStrategy.isPreemptionActive() && (numTasksToPreempt > 0 || missingResumableSlots > 0)) { if (!sizeBasedJobsDescIter.hasNext()) { if (LOG.isDebugEnabled()) { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "):" + " should preempt " + numTasksToPreempt + " for new tasks and " + missingResumableSlots + " for resumable " + "tasks but no sizeBasedJob is running"); } break; } Entry<JobDurationInfo, JobInProgress> nextSBJ = sizeBasedJobsDescIter.next(); JobInProgress jipToPreempt = nextSBJ.getValue(); /* don't try to suspend if jip is bigger than any other jip */ if (jdi != null) { if (jipToPreempt.getJobID().equals(jip.getJobID())) { return new ClaimedSlots(startingNumTasksToPreemptForNew - numTasksToPreempt, startingResumableSlots - missingResumableSlots); } if (JOB_DURATION_COMPARATOR.compare(nextSBJ.getKey(), jdi) <= 0) { if (LOG.isDebugEnabled()) { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "):" + " should preempt " + numTasksToPreempt + ", but bigger job avail is " + jip.getJobID() + ".len: " + jdi.getPhaseDuration() + " > " + nextSBJ.getValue().getJobID() + ".len: " + nextSBJ.getKey().getPhaseDuration()); } return new ClaimedSlots(startingNumTasksToPreemptForNew - numTasksToPreempt, startingResumableSlots - missingResumableSlots); } } if (jipToPreempt.getJobID().equals(jip.getJobID())) { continue; } /* * don't try to claim slots from a job in training * * FIXME: ideally a job can claim slots from a training job until this job * has enough tasks for training */ if (!this.isTrained(jipToPreempt, type)) { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "):" + " ignoring " + jipToPreempt.getJobID() + " because in training"); continue; } int numSuspendedOnThisTT = 0; /* if jipToPreempt has tasks on this TT, then suspend them */ if (biggerOnTT != null // && type == TaskType.REDUCE && biggerOnTT.getKey().getJobID().equals(nextSBJ.getKey().getJobID())) { TreeMap<TaskAttemptID, TaskStatus> preemptableTAIDS = biggerOnTT.getValue().taskStatuses; int numPreemptions = Math.min(preemptableTAIDS.size(), missingResumableSlots + numTasksToPreempt); for (int i = 0; i < numPreemptions; i++) { TaskAttemptID pTAID = preemptableTAIDS.firstKey(); TaskStatus pTS = preemptableTAIDS.remove(pTAID); JobInProgress pJIP = this.taskTrackerManager.getJob(pTAID.getJobID()); TaskInProgress pTIP = pJIP.getTaskInProgress(pTAID.getTaskID()); if (type == TaskType.REDUCE) { // if (this.eagerPreemption == PreemptionType.KILL // && pTIP.killTask(pTAID, false)) { // if (missingResumableSlots > 0) // missingResumableSlots -= 1; // else // numTasksToPreempt -= 1; // numSuspendedOnThisTT += 1; // if (jdi == null) { // taskHelper.kill(pTAID, jip.getJobID(), phase); // } else { // taskHelper.kill(pTAID, jip.getJobID(), phase, nextSBJ.getKey(), // jdi); // } // } else if (this.preemptionStrategy.isPreemptionActive() // && this.canBeSuspended(pTS) && pTIP.suspendTaskAttempt(pTAID)) { if (this.preemptionStrategy.isPreemptionActive() && this.preemptionStrategy.canBePreempted(pTS) && this.preemptionStrategy.preempt(pTIP, pTS)) { if (missingResumableSlots > 0) missingResumableSlots -= 1; else numTasksToPreempt -= 1; numSuspendedOnThisTT += 1; if (jdi == null) { taskHelper.suspend(pTAID, jip.getJobID(), phase); } else { taskHelper.suspend(pTAID, jip.getJobID(), phase, nextSBJ.getKey(), jdi); } } else { LOG.debug(phase.toString() + "(" + jip.getJobID() + ":" + type + "): cannot suspend " + pTAID + " for " + jip); } } } if (preemptableTAIDS.size() - numPreemptions <= 0) { biggerOnTT = sizeBasedJobsDescIterOnTT.hasNext() ? sizeBasedJobsDescIterOnTT.next() : null; } } /* #tasks that can be preempted */ int numPreemptibleRunTasks = this.getNumRunningTasks(jipToPreempt, type) - numSuspendedOnThisTT; /* * Two cases: numToSkip is bigger then preemptible tasks or it is not: - * is bigger: then we skip this preemptible jip - it is not: then * numToSkip is set to 0 and we do the real wait preemption */ if (numPreemptibleRunTasks <= numToSkip) { numToSkip -= numPreemptibleRunTasks; } else { /* #tasks that can be preempted by jip */ int numPreemptibleByJIPRunTasks = numPreemptibleRunTasks - numToSkip; numToSkip = 0; /* #tasks that will be preempted by jip on other TTs s */ int numRunTasksEventuallyPreemptedByJIP = Math.min(numTasksToPreempt, numPreemptibleByJIPRunTasks); numTasksToPreempt -= numRunTasksEventuallyPreemptedByJIP; } } return new ClaimedSlots(startingNumTasksToPreemptForNew - numTasksToPreempt, startingResumableSlots - missingNewSlots); }