List of usage examples for java.util LinkedList addFirst
public void addFirst(E e)
From source file:com.moviejukebox.plugin.KinopoiskPlugin.java
/** * Scan Kinopoisk html page for the specified movie */// w ww .ja v a 2 s .c o m private boolean updateKinopoiskMediaInfo(Movie movie, String kinopoiskId) { try { String originalTitle = movie.getTitle(); String newTitle = originalTitle; String xml = httpClient.request(FILM_URL + kinopoiskId, CHARSET); boolean etalonFlag = kinopoiskId.equals(etalonId); // Work-around for issue #649 xml = xml.replace("…", "…"); xml = xml.replace("—", "—"); // Title boolean overrideTitle = OverrideTools.checkOverwriteTitle(movie, KINOPOISK_PLUGIN_ID); if (overrideTitle || etalonFlag) { newTitle = HTMLTools.extractTag(xml, "class=\"moviename-big\" itemprop=\"name\">", 0, "</"); if (StringTools.isValidString(newTitle)) { if (overrideTitle) { int i = newTitle.indexOf("(?"); if (i >= 0) { newTitle = newTitle.substring(0, i); movie.setMovieType(Movie.TYPE_TVSHOW); } newTitle = newTitle.replace('\u00A0', ' ').trim(); if (movie.getSeason() != -1) { newTitle = newTitle + ", ? " + movie.getSeason(); } // Original title originalTitle = newTitle; for (String s : HTMLTools.extractTags(xml, "class=\"moviename-big\" itemprop=\"name\">", "</span>", "itemprop=\"alternativeHeadline\">", "</span>")) { if (!s.isEmpty()) { originalTitle = s; if (joinTitles) { newTitle = newTitle + Movie.SPACE_SLASH_SPACE + originalTitle; } } break; } } else { newTitle = originalTitle; } } else { if (etalonFlag) { LOG.error(SITE_DESIGN, "movie title"); } newTitle = originalTitle; } } // Plot if (OverrideTools.checkOverwritePlot(movie, KINOPOISK_PLUGIN_ID)) { StringBuilder plot = new StringBuilder(); for (String subPlot : HTMLTools.extractTags(xml, "<span class=\"_reachbanner_\"", "</span>", "", "<")) { if (!subPlot.isEmpty()) { if (plot.length() > 0) { plot.append(" "); } plot.append(subPlot); } } movie.setPlot(plot.toString(), KINOPOISK_PLUGIN_ID); if (etalonFlag && (plot.length() == 0)) { LOG.error(SITE_DESIGN, "plot"); } } boolean valueFounded = false; boolean genresFounded = false; boolean certificationFounded = false; boolean countryFounded = false; boolean yearFounded = false; boolean runtimeFounded = false; boolean taglineFounded = false; boolean releaseFounded = false; for (String item : HTMLTools.extractTags(xml, "<table class=\"info\">", "</table>", "<tr", "</tr>")) { item = "<tr>" + item + "</tr>"; // Genres if (OverrideTools.checkOverwriteGenres(movie, KINOPOISK_PLUGIN_ID)) { LinkedList<String> newGenres = new LinkedList<>(); boolean innerGenresFound = false; for (String genre : HTMLTools.extractTags(item, "><", "</tr>", "<a href=\"/lists/m_act%5Bgenre%5D/", "</a>")) { innerGenresFound = true; genre = genre.substring(0, 1).toUpperCase() + genre.substring(1, genre.length()); if ("".equalsIgnoreCase(genre)) { newGenres.addFirst(genre); } else { newGenres.add(genre); } } if (innerGenresFound) { // Limit genres count int maxGenres = 9; try { maxGenres = PropertiesUtil.getIntProperty("genres.max", 9); } catch (Exception ignore) { // } while (newGenres.size() > maxGenres) { newGenres.removeLast(); } movie.setGenres(newGenres, KINOPOISK_PLUGIN_ID); genresFounded = true; } } // Certification from MPAA if (OverrideTools.checkOverwriteCertification(movie, KINOPOISK_PLUGIN_ID)) { if (!certificationFounded) { certificationFounded = getCertification(movie, item, "> MPAA<", "</tr>", "<a href=\"/film", "</a>", "alt=\" "); } if (!certificationFounded || russianCertification) { certificationFounded |= getCertification(movie, item, ">?<", "</tr>", "<td style=\"color", "</span>", "ageLimit "); } } // Country if (OverrideTools.checkOverwriteCountry(movie, KINOPOISK_PLUGIN_ID)) { Collection<String> scraped = HTMLTools.extractTags(item, ">?<", "</tr>", "a href=\"/lists/m_act%5Bcountry%5D/", "</a>"); if (scraped != null && !scraped.isEmpty()) { List<String> countries = new ArrayList<>(); for (String country : scraped) { if (translitCountry) { country = FileTools.makeSafeFilename(country); } countries.add(country); if (!countryAll) { // just first country, so break here break; } } movie.setCountries(countries, KINOPOISK_PLUGIN_ID); countryFounded = true; } } // Year if (OverrideTools.checkOverwriteYear(movie, KINOPOISK_PLUGIN_ID)) { for (String year : HTMLTools.extractTags(item, "><", "</tr>", "<a href=\"/lists/m_act%5Byear%5D/", "</a>")) { movie.setYear(year, KINOPOISK_PLUGIN_ID); yearFounded = true; break; } } // Run time if (OverrideTools.checkOverwriteRuntime(movie, KINOPOISK_PLUGIN_ID) || etalonFlag) { for (String runtime : HTMLTools.extractTags(item, ">?<", "</tr>", "<td", "</td>")) { if (runtime.contains("<span")) { runtime = runtime.substring(0, runtime.indexOf("<span")); } movie.setRuntime(runtime, KINOPOISK_PLUGIN_ID); runtimeFounded = true; break; } } // Tagline if (OverrideTools.checkOverwriteTagline(movie, KINOPOISK_PLUGIN_ID)) { for (String tagline : HTMLTools.extractTags(item, ">?<", "</tr>", "<td ", "</td>")) { if (tagline.length() > 0) { movie.setTagline(tagline.replace("\u00AB", "\"").replace("\u00BB", "\""), KINOPOISK_PLUGIN_ID); taglineFounded = true; break; } } } // Release date if (OverrideTools.checkOverwriteReleaseDate(movie, KINOPOISK_PLUGIN_ID)) { String releaseDate = ""; for (String release : HTMLTools.extractTags(item, "> ()<", "</tr>", "<a href=\"/film/", "</a>")) { releaseDate = release; releaseFounded = true; break; } if (StringUtils.isBlank(releaseDate)) { for (String release : HTMLTools.extractTags(item, "> ()<", "</tr>", "<a href=\"/film/", "</a>")) { releaseDate = release; releaseFounded = true; break; } } movie.setReleaseDate(releaseDate, KINOPOISK_PLUGIN_ID); } } if (etalonFlag) { if (!genresFounded) { LOG.error(SITE_DESIGN, "genres"); } if (!certificationFounded) { LOG.error(SITE_DESIGN, "certification"); } if (!countryFounded) { LOG.error(SITE_DESIGN, "country"); } if (!yearFounded) { LOG.error(SITE_DESIGN, "year"); } if (!runtimeFounded) { LOG.error(SITE_DESIGN, "runtime"); } if (!taglineFounded) { LOG.error(SITE_DESIGN, "tagline"); } if (!releaseFounded) { LOG.error(SITE_DESIGN, "release"); } } // Rating if (!nfoRating) { valueFounded = false; for (String rating : HTMLTools.extractTags(xml, "<a href=\"/film/" + kinopoiskId + "/votes/\"", "</a>", "<span", "</span>")) { int kinopoiskRating = StringTools.parseRating(rating); movie.addRating(KINOPOISK_PLUGIN_ID, kinopoiskRating); valueFounded = true; break; } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "rating"); } int imdbRating = movie.getRating(IMDB_PLUGIN_ID); if (imdbRating == -1 || etalonFlag) { // Get IMDB rating from kinopoisk page String rating = HTMLTools.extractTag(xml, ">IMDb:", 0, " ("); valueFounded = false; if (StringTools.isValidString(rating)) { imdbRating = StringTools.parseRating(rating); movie.addRating(IMDB_PLUGIN_ID, imdbRating); valueFounded = true; } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "IMDB rating"); } } } // Top250 if (OverrideTools.checkOverwriteTop250(movie, KINOPOISK_PLUGIN_ID)) { movie.setTop250(HTMLTools.extractTag(xml, "<a href=\"/level/20/#", 0, "\""), KINOPOISK_PLUGIN_ID); } // Poster String posterURL = movie.getPosterURL(); if (StringTools.isNotValidString(posterURL) || (!nfoPoster && poster) || etalonFlag) { valueFounded = false; if (poster || etalonFlag) { String previousURL = posterURL; posterURL = Movie.UNKNOWN; // Load page with all poster String wholeArts = httpClient.request(FILM_URL + kinopoiskId + "/posters/", CHARSET); if (StringTools.isValidString(wholeArts)) { if (wholeArts.contains("<table class=\"fotos")) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/poster/sm_", 0, "\""); if (StringTools.isValidString(picture)) { posterURL = "http://st.kinopoisk.ru/images/poster/" + picture; valueFounded = true; } } } if (StringTools.isNotValidString(posterURL)) { posterURL = previousURL; } if (StringTools.isValidString(posterURL)) { movie.setPosterURL(posterURL); movie.setPosterFilename(movie.getBaseName() + ".jpg"); LOG.debug("Set poster URL to {} for {}", posterURL, movie.getBaseName()); } } if (StringTools.isNotValidString(movie.getPosterURL())) { if (overrideTitle) { movie.setTitle(originalTitle, KINOPOISK_PLUGIN_ID); } // Removing Poster info from plugins. Use of PosterScanner routine instead. // movie.setPosterURL(locatePosterURL(movie, "")); } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "poster"); } } // Fanart String fanURL = movie.getFanartURL(); if (StringTools.isNotValidString(fanURL) || (!nfoFanart && (fanArt || kadr)) || etalonFlag || fanURL.contains("originalnull")) { valueFounded = false; try { String previousURL = fanURL; fanURL = Movie.UNKNOWN; // Load page with all wallpaper String wholeArts = httpClient.request("http://www.kinopoisk.ru/film/" + kinopoiskId + "/wall/", CHARSET); if (StringTools.isValidString(wholeArts)) { if (wholeArts.contains("<table class=\"fotos")) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/wallpaper/sm_", 0, ".jpg"); if (StringTools.isValidString(picture)) { if (picture.contains("_")) { picture = picture.substring(0, picture.indexOf('_')); } String size = HTMLTools.extractTag(wholeArts, "<u><a href=\"/picture/" + picture + "/w_size/", 0, "/"); wholeArts = httpClient.request( "http://www.kinopoisk.ru/picture/" + picture + "/w_size/" + size, CHARSET); if (StringTools.isValidString(wholeArts)) { picture = HTMLTools.extractTag(wholeArts, "src=\"http://st-im.kinopoisk.r/im/wallpaper/", 0, "\""); if (StringTools.isValidString(picture)) { fanURL = "http://st.kinopoisk.ru/im/wallpaper/" + picture; valueFounded = true; } } } } } if (kadr && (StringTools.isNotValidString(fanURL) || fanURL.contains("originalnull"))) { // Load page with all videoimage wholeArts = httpClient.request(FILM_URL + kinopoiskId + "/stills/", CHARSET); if (StringTools.isValidString(wholeArts)) { // Looking for photos table int photosInd = wholeArts.indexOf("<table class=\"fotos"); if (photosInd != -1) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/kadr/sm_", 0, "\""); if (StringTools.isValidString(picture)) { fanURL = "http://www.kinopoisk.ru/images/kadr/" + picture; valueFounded = true; } } } } if (StringTools.isNotValidString(fanURL)) { fanURL = previousURL; } if (StringTools.isValidString(fanURL)) { movie.setFanartURL(fanURL); movie.setFanartFilename(movie.getBaseName() + fanartToken + "." + fanartExtension); LOG.debug("Set fanart URL to {} for {}", fanURL, movie.getBaseName()); } } catch (IOException ex) { // Ignore } } // Studio/Company if (OverrideTools.checkOverwriteCompany(movie, KINOPOISK_PLUGIN_ID)) { xml = httpClient.request(FILM_URL + kinopoiskId + "/studio/", CHARSET); valueFounded = false; if (StringTools.isValidString(xml)) { Collection<String> studio = new ArrayList<>(); if (xml.contains(">?:<")) { for (String tmp : HTMLTools.extractTags(xml, ">?:<", "</table>", "<a href=\"/lists/m_act%5Bstudio%5D/", "</a>")) { studio.add(HTMLTools.removeHtmlTags(tmp)); if (!companyAll) { break; } } } if (xml.contains(">:<") && (companyAll || studio.isEmpty())) { for (String tmp : HTMLTools.extractTags(xml, ">:<", "</table>", "<a href=\"/lists/m_act%5Bcompany%5D/", "</a>")) { studio.add(HTMLTools.removeHtmlTags(tmp)); } } if (!studio.isEmpty()) { movie.setCompany(companyAll ? StringUtils.join(studio, Movie.SPACE_SLASH_SPACE) : new ArrayList<>(studio).get(0), KINOPOISK_PLUGIN_ID); valueFounded = true; } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "company"); } } // Awards if ((scrapeAwards && !nfoAwards) || etalonFlag) { if (clearAward) { movie.clearAwards(); } xml = httpClient.request("http://www.kinopoisk.ru/film/" + kinopoiskId + "/awards/", CHARSET); Collection<AwardEvent> awards = new ArrayList<>(); if (StringTools.isValidString(xml)) { int beginIndex = xml.indexOf("<b><a href=\"/awards/"); if (beginIndex != -1) { for (String item : HTMLTools.extractTags(xml, "<table cellspacing=0 cellpadding=0 border=0 width=100%>", "<br /><br /><br /><br /><br /><br />", "<table cellspacing=\"0\" cellpadding=\"0\" border=\"0\" width=\"100%\" ", "</table>")) { String name = Movie.UNKNOWN; int year = -1; int won = 0; int nominated = 0; Collection<String> wons = new ArrayList<>(); Collection<String> nominations = new ArrayList<>(); for (String tmp : HTMLTools.extractTags(item, "<td height=\"40\" class=\"news\" style=\"padding: 10px\">", "</td>", "<a href=\"/awards/", "</a>")) { int coma = tmp.indexOf(","); name = tmp.substring(0, coma); year = NumberUtils.toInt(tmp.substring(coma + 2, coma + 6)); break; } for (String tmp : HTMLTools.extractTags(item, "><", ":", "(", ")")) { won = NumberUtils.toInt(tmp); break; } if (won > 0) { for (String tmp : HTMLTools.extractTags(item, "><", "</ul>", "<li class=\"trivia\">", "</li>")) { wons.add(HTMLTools.removeHtmlTags(tmp).replaceAll(" {2,}", " ")); } } for (String tmp : HTMLTools.extractTags(item, ">?<", ":", "(", ")")) { nominated = NumberUtils.toInt(tmp); break; } if (nominated > 0) { for (String tmp : HTMLTools.extractTags(item, ">?<", "</ul>", "<li class=\"trivia\"", "</li>")) { nominations.add(HTMLTools.removeHtmlTags(tmp).replaceAll(" {2,}", " ")); } } if (StringTools.isValidString(name) && year > 1900 && year < 2020 && (!scrapeWonAwards || (won > 0))) { Award award = new Award(); award.setName(name); award.setYear(year); if (won > 0) { award.setWons(wons); } if (nominated > 0) { award.setNominations(nominations); } AwardEvent event = new AwardEvent(); event.setName(name); event.addAward(award); awards.add(event); } } } } if (!awards.isEmpty()) { movie.setAwards(awards); } else if (etalonFlag) { LOG.error(SITE_DESIGN, "award"); } } // Cast enhancement boolean overrideCast = OverrideTools.checkOverwriteActors(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleCast = OverrideTools.checkOverwritePeopleActors(movie, KINOPOISK_PLUGIN_ID); boolean overrideDirectors = OverrideTools.checkOverwriteDirectors(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleDirectors = OverrideTools.checkOverwritePeopleDirectors(movie, KINOPOISK_PLUGIN_ID); boolean overrideWriters = OverrideTools.checkOverwriteWriters(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleWriters = OverrideTools.checkOverwritePeopleWriters(movie, KINOPOISK_PLUGIN_ID); if (overrideCast || overridePeopleCast || overrideDirectors || overridePeopleDirectors || overrideWriters || overridePeopleWriters || etalonFlag) { xml = httpClient.request(FILM_URL + kinopoiskId + "/cast", CHARSET); if (StringTools.isValidString(xml)) { if (overrideDirectors || overridePeopleDirectors || etalonFlag) { int count = scanMoviePerson(movie, xml, "director", directorMax, overrideDirectors, overridePeopleDirectors); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "directors"); } } if (overrideWriters || overridePeopleWriters || etalonFlag) { int count = scanMoviePerson(movie, xml, "writer", writerMax, overrideWriters, overridePeopleWriters); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "writers"); } } if (overrideCast || overridePeopleCast || etalonFlag) { int count = scanMoviePerson(movie, xml, "actor", actorMax, overrideCast, overridePeopleCast); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "cast"); } } Collection<Filmography> outcast = new ArrayList<>(); for (Filmography p : movie.getPeople()) { if (StringTools.isNotValidString(p.getId(KINOPOISK_PLUGIN_ID))) { outcast.add(p); } } for (Filmography p : outcast) { movie.removePerson(p); } } } // Business if (scrapeBusiness || etalonFlag) { xml = httpClient.request(FILM_URL + kinopoiskId + "/box/", CHARSET); if (StringTools.isValidString(xml)) { valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, ">:<", "</table>", "<font color=\"#ff6600\"", "</h3>")) { if (StringTools.isValidString(tmp)) { tmp = tmp.replaceAll("\u00A0", ",").replaceAll(",$", ""); if (StringTools.isValidString(tmp) && !"--".equals(tmp)) { movie.setBudget(tmp); valueFounded = true; break; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: summary"); } valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, "> -? (?)<", "</table>", "<h3 style=\"font-size: 18px; margin: 0; padding: 0;color:#f60\"", "</h3>")) { if (StringTools.isValidString(tmp)) { tmp = tmp.replaceAll("\u00A0", ",").replaceAll(",$", ""); if (StringTools.isValidString(tmp) && !"--".equals(tmp)) { movie.setOpenWeek("USA", tmp); valueFounded = true; break; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: first weekend"); } valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, ">?? ?<", "</table>", "<tr><td colspan=2", "</h3>")) { tmp += "</h3>"; String country = HTMLTools.extractTag(tmp, ">", ":"); if (StringTools.isValidString(country)) { String money = HTMLTools .removeHtmlTags("<h3 " + HTMLTools.extractTag(tmp, "<h3 ", "</h3>")) .replaceAll("\u00A0", ",").replaceAll(",$", ""); if (!"--".equals(money)) { movie.setGross( " ??" .equals(country) ? "Russia" : " ?".equals(country) ? "USA" : " ?".equals(country) ? "Worldwide" : " ?".equals( country) ? "Others" : country, money); valueFounded = true; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: gross"); } } else if (etalonFlag) { LOG.error(SITE_DESIGN, "business"); } } // Did You Know if (scrapeTrivia || etalonFlag) { if (clearTrivia) { movie.clearDidYouKnow(); } if (etalonFlag && triviaMax == 0) { triviaMax = 1; } if (triviaMax != 0) { xml = httpClient.request(FILM_URL + kinopoiskId + "/view_info/ok/#trivia", CHARSET); if (StringTools.isValidString(xml)) { int i = 0; for (String tmp : HTMLTools.extractTags(xml, "> , ...<", "</ul>", "<li class=\"trivia", "</li>")) { if (i < triviaMax || triviaMax == -1) { movie.addDidYouKnow(HTMLTools.removeHtmlTags(tmp)); valueFounded = true; i++; } else { break; } } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "trivia"); } } // Finally set title if (overrideTitle) { movie.setTitle(newTitle, KINOPOISK_PLUGIN_ID); } } catch (IOException | NumberFormatException error) { LOG.error("Failed retreiving movie data from Kinopoisk : {}", kinopoiskId); LOG.error(SystemTools.getStackTrace(error)); } return true; }
From source file:org.apache.hyracks.algebricks.rewriter.rules.PushSelectIntoJoinRule.java
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>(); Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>(); Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>(); Collection<LogicalVariable> liveInOpsToPushRight = new HashSet<LogicalVariable>(); List<ILogicalOperator> pushedOnLeft = new ArrayList<ILogicalOperator>(); List<ILogicalOperator> pushedOnRight = new ArrayList<ILogicalOperator>(); List<ILogicalOperator> pushedOnEither = new ArrayList<ILogicalOperator>(); LinkedList<ILogicalOperator> notPushedStack = new LinkedList<ILogicalOperator>(); Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>(); Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>(); AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.SELECT) { return false; }// ww w . j av a 2s. c om SelectOperator select = (SelectOperator) op; Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0); AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue(); AbstractLogicalOperator op2 = son; boolean needToPushOps = false; while (son.isMap()) { needToPushOps = true; Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0); son = (AbstractLogicalOperator) opRefLink.getValue(); } if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN && son.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) { return false; } boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN; AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son; Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0); Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1); if (needToPushOps) { ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue(); ILogicalOperator joinBranchRight = joinBranchRightRef.getValue(); VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft); VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight); Mutable<ILogicalOperator> opIterRef = opRef2; ILogicalOperator opIter = op2; while (opIter != join) { LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag(); if (tag == LogicalOperatorTag.PROJECT) { notPushedStack.addFirst(opIter); } else { VariableUtilities.getUsedVariables(opIter, usedVars); VariableUtilities.getProducedVariables(opIter, producedVars); if (usedVars.size() == 0) { pushedOnEither.add(opIter); } else if (joinLiveVarsLeft.containsAll(usedVars)) { pushedOnLeft.add(opIter); liveInOpsToPushLeft.addAll(producedVars); } else if (joinLiveVarsRight.containsAll(usedVars)) { pushedOnRight.add(opIter); liveInOpsToPushRight.addAll(producedVars); } else { return false; } } opIterRef = opIter.getInputs().get(0); opIter = opIterRef.getValue(); } if (isLoj && pushedOnLeft.isEmpty()) { return false; } } boolean intersectsAllBranches = true; boolean[] intersectsBranch = new boolean[join.getInputs().size()]; LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>(); select.getCondition().getValue().getUsedVariables(selectVars); int i = 0; for (Mutable<ILogicalOperator> branch : join.getInputs()) { LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>(); VariableUtilities.getLiveVariables(branch.getValue(), branchVars); if (i == 0) { branchVars.addAll(liveInOpsToPushLeft); } else { branchVars.addAll(liveInOpsToPushRight); } if (OperatorPropertiesUtil.disjoint(selectVars, branchVars)) { intersectsAllBranches = false; } else { intersectsBranch[i] = true; } i++; } if (!intersectsBranch[0] && !intersectsBranch[1]) { return false; } if (needToPushOps) { //We should push independent ops into the first branch that the selection depends on if (intersectsBranch[0]) { pushOps(pushedOnEither, joinBranchLeftRef, context); } else { pushOps(pushedOnEither, joinBranchRightRef, context); } pushOps(pushedOnLeft, joinBranchLeftRef, context); pushOps(pushedOnRight, joinBranchRightRef, context); } if (intersectsAllBranches) { addCondToJoin(select, join, context); } else { // push down Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator(); ILogicalExpression selectCondition = select.getCondition().getValue(); boolean lojToInner = false; for (int j = 0; j < intersectsBranch.length; j++) { Mutable<ILogicalOperator> branch = branchIter.next(); boolean inter = intersectsBranch[j]; if (!inter) { continue; } // if a left outer join, if the select condition is not-missing filtering, // we rewrite left outer join // to inner join for this case. if (j > 0 && isLoj && containsNotMissingFiltering(selectCondition)) { lojToInner = true; } if ((j > 0 && isLoj) && containsMissingFiltering(selectCondition)) { // Select "is-not-missing($$var)" cannot be pushed in the right branch of a LOJ; notPushedStack.addFirst(select); } else { // Conditions for the left branch can always be pushed. // Other conditions can be pushed to the right branch of a LOJ. copySelectToBranch(select, branch, context); } } if (lojToInner) { // Rewrites left outer join to inner join. InnerJoinOperator innerJoin = new InnerJoinOperator(join.getCondition()); innerJoin.getInputs().addAll(join.getInputs()); join = innerJoin; context.computeAndSetTypeEnvironmentForOperator(join); } } ILogicalOperator top = join; for (ILogicalOperator npOp : notPushedStack) { List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs(); npInpList.clear(); npInpList.add(new MutableObject<ILogicalOperator>(top)); context.computeAndSetTypeEnvironmentForOperator(npOp); top = npOp; } opRef.setValue(top); return true; }
From source file:com.projity.script.object.TimeIntervals.java
public TimeIntervals translate(int winCount) { //TODO case winCount<0 // for (TimeWindow w : history) System.out.println("history0: "+w); // for (TimeWindow w : win) System.out.println("win0: "+w); //for (TimeWindow w : history) System.out.println("id="+w.getId()); TimeIntervals t = new TimeIntervals(); t.setScale(scale);/* ww w . j a v a2 s . c o m*/ LinkedList<TimeWindow> twin = t.getWin(); if (winCount == 0 || win.size() == 0) return t; //or null if (winCount > 0) { t.winId = winId + win.size(); int lastId = t.winId - 1 + winCount; int maxHistoryId = Math.min(history.getLast().getId(), lastId); int i = t.winId; if (i <= maxHistoryId) { ListIterator<TimeWindow> it = history.listIterator(); TimeWindow w; while (it.hasNext()) { w = it.next(); if (w.getId() == t.winId) { it.previous(); break; } } for (; i <= maxHistoryId && it.hasNext(); i++) { w = it.next(); twin.add(w); // System.out.println("Found in history: "+w); } } LinkedList<TimeWindow> newWin = new LinkedList<TimeWindow>(); generateWindows(scale, (twin.size() > 0 ? twin : win).getLast().getE(), start, end, lastId - i + 1, newWin); t.indexWindows(t.winId + t.getWin().size(), newWin); // for (TimeWindow w : newWin) System.out.println("New window: "+w); t.getWin().addAll(newWin); history.addAll(newWin); } else { t.winId = winId - 1; int lastId = t.winId + 1 + winCount; int minHistoryId = Math.max(history.getFirst().getId(), lastId); int i = t.winId; if (i >= minHistoryId) { ListIterator<TimeWindow> it = history.listIterator(history.size() - 1); TimeWindow w; while (it.hasPrevious()) { w = it.previous(); if (w.getId() == t.winId) { it.next(); break; } } for (; i >= minHistoryId; i--) { w = it.previous(); twin.addFirst(w); // System.out.println("Found in history: "+w); } } // System.out.println("winId="+winId+", t.winId="+t.winId+", lastId="+lastId+", i="+i+" minHistoryId="+minHistoryId); LinkedList<TimeWindow> newWin = new LinkedList<TimeWindow>(); generateWindows(scale, (twin.size() > 0 ? twin : win).getFirst().getS(), start, end, lastId - i - 1, newWin); t.indexWindows(lastId, newWin); // for (TimeWindow w : newWin) System.out.println("New window: "+w); t.getWin().addAll(0, newWin); history.addAll(0, newWin); } int translation = 0; for (TimeWindow w : t.getWin()) { if (winCount > 0) { win.removeFirst(); win.addLast(w); translation++; } else { win.removeLast(); win.addFirst(w); translation--; } } winId = winId + translation; t.setTranslation(translation); // for (TimeWindow w : history) System.out.println("history1: "+w); // for (TimeWindow w : win) System.out.println("win1: "+w); // for (TimeWindow w : twin) System.out.println("t.win1: "+w); return t; }
From source file:org.drools.semantics.builder.model.inference.DelegateInferenceStrategy.java
private void applyPropertyRestrictions(OWLOntology ontoDescr, OntoModel hierarchicalModel, OWLDataFactory factory) {/*w ww .j av a2 s . c o m*/ for (PropertyRelation prop : hierarchicalModel.getProperties()) { if (prop.getTarget() == null) { logger.warn("Property without target concept " + prop.getName()); } } Map<String, Set<OWLClassExpression>> supers = new HashMap<String, Set<OWLClassExpression>>(); for (OWLClass klass : ontoDescr.getClassesInSignature(true)) { supers.put(klass.getIRI().toQuotedString(), new HashSet<OWLClassExpression>()); } for (OWLClass klass : ontoDescr.getClassesInSignature(true)) { if (isDelegating(klass)) { OWLClassExpression delegate = aliases.get(klass); supers.get(delegate.asOWLClass().getIRI().toQuotedString()) .addAll(klass.getSuperClasses(ontoDescr)); } else { Set<OWLClassExpression> sup = supers.get(klass.asOWLClass().getIRI().toQuotedString()); Set<OWLClassExpression> ancestors = klass.getSuperClasses(ontoDescr); sup.addAll(ancestors); for (OWLClassExpression anc : ancestors) { if (reverseAliases.containsKey(anc)) { sup.addAll(reverseAliases.get(anc)); } } } } for (Concept con : hierarchicalModel.getConcepts()) { //use concepts as they're sorted! if (isDelegating(con.getIri())) { continue; } if (con == null) { logger.warn("Looking for superclasses of an undefined concept"); continue; } LinkedList<OWLClassExpression> orderedSupers = new LinkedList<OWLClassExpression>(); // cardinalities should be fixed last for (OWLClassExpression sup : supers.get(con.getIri())) { if (sup instanceof OWLCardinalityRestriction) { orderedSupers.addLast(sup); } else { orderedSupers.addFirst(sup); } } for (OWLClassExpression sup : orderedSupers) { if (sup.isClassExpressionLiteral()) { continue; } processSuperClass(sup, con, hierarchicalModel, factory); } // for ( PropertyRelation prop : con.getProperties().values() ) { // if ( prop.isRestricted() && ( prop.getMaxCard() == null || prop.getMaxCard() > 1 ) ) { // prop.setName( prop.getName() + "s" ); // } // } } }
From source file:de.tudarmstadt.ukp.wikipedia.parser.mediawiki.ModularParser.java
private Table buildTable(SpanManager sm, ContentElementParsingParameters cepp, LinkedList<Span> lineSpans) { Table result = new Table(); int col = -1; int row = 0;//from w w w . j a v a2 s .co m int subTables = 0; LinkedList<Span> tableDataSpans = new LinkedList<Span>(); sm.manageList(tableDataSpans); if (calculateSrcSpans) { result.setSrcSpan(new SrcSpan(sm.getSrcPos(lineSpans.getFirst().getStart()), -1)); } lineSpans.removeFirst(); while (!lineSpans.isEmpty()) { Span s = lineSpans.removeFirst(); int pos = s.nonWSCharPos(sm); char c0 = s.charAt(pos, sm); char c1 = s.charAt(pos + 1, sm); if (subTables == 0 && (c0 == '!' || c0 == '|')) { if (!tableDataSpans.isEmpty()) { lineSpans.addFirst(s); SrcSpan ei = null; if (calculateSrcSpans) { ei = new SrcSpan(sm.getSrcPos(tableDataSpans.getFirst().getStart() - 1) + 1, -1); } TableElement te = new TableElement(parseSections(sm, cepp, tableDataSpans), row, col); te.setSrcSpan(ei); result.addTableElement(te); lineSpans.removeFirst(); } col++; if (c1 == '-') { row++; col = -1; continue; } else if (c0 == '|' && c1 == '}') { sm.removeManagedList(tableDataSpans); if (calculateSrcSpans) { result.getSrcSpan().setEnd(sm.getSrcPos(s.getEnd())); } return result; } else if (c0 == '|' && c1 == '+') { result.setTitleElement( parseContentElement(sm, cepp, new Span(s.getStart() + pos + 2, s.getEnd()).trim(sm))); continue; } else { int multipleCols; if ((multipleCols = sm.indexOf("||", s.getStart() + pos + 1, s.getEnd())) != -1) { lineSpans.addFirst(new Span(multipleCols + 1, s.getEnd())); s.setEnd(multipleCols); } int optionTagPos = sm.indexOf("|", s.getStart() + pos + 1, s.getEnd()); if (optionTagPos != -1) { s.setStart(optionTagPos + 1).trim(sm); } else { s.adjustStart(pos + 1).trim(sm); } } } else if (c0 == '|' && c1 == '}') { subTables--; } else if (c0 == '{' && c1 == '|') { subTables++; } tableDataSpans.addLast(s); } if (tableDataSpans.size() != 0) { SrcSpan ei = null; if (calculateSrcSpans) { ei = new SrcSpan(sm.getSrcPos(tableDataSpans.getFirst().getStart() - 1) + 1, -1); } TableElement te = new TableElement(parseSections(sm, cepp, tableDataSpans), row, col); te.setSrcSpan(ei); result.addTableElement(te); } sm.removeManagedList(tableDataSpans); if (calculateSrcSpans) { result.getSrcSpan().setEnd(-1); } return result; }
From source file:org.hyperic.hq.measurement.server.session.AvailabilityManagerImpl.java
private PageList<HighLowMetricValue> getPageList(List<AvailabilityDataRLE> availInfo, long begin, long end, long interval, boolean prependUnknowns) { PageList<HighLowMetricValue> rtn = new PageList<HighLowMetricValue>(); for (Iterator<AvailabilityDataRLE> it = availInfo.iterator(); it.hasNext();) { AvailabilityDataRLE rle = it.next(); long availStartime = rle.getStartime(); long availEndtime = rle.getEndtime(); //skip measurements that are before first time slot if (availEndtime < begin) { continue; }// w w w. j a v a 2s. c o m LinkedList<AvailabilityDataRLE> queue = new LinkedList<AvailabilityDataRLE>(); queue.add(rle); int i = 0; for (long curr = begin; curr <= end; curr += interval) { long next = curr + interval; next = (next > end) ? end : next; long endtime = ((AvailabilityDataRLE) queue.getFirst()).getEndtime(); //while next time slot is after measurement time range while (next > endtime) { // it should not be the case that there are no more // avails in the array, but we need to handle it if (it.hasNext()) { AvailabilityDataRLE tmp = (AvailabilityDataRLE) it.next(); queue.addFirst(tmp); endtime = tmp.getEndtime(); } else { endtime = availEndtime; int measId = rle.getMeasurement().getId().intValue(); String msg = "Measurement, " + measId + ", for interval " + begin + " - " + end + " did not return a value for range " + curr + " - " + (curr + interval); log.warn(msg); } } endtime = availEndtime; while (curr > endtime) { queue.removeLast(); // this should not happen unless the above !it.hasNext() // else condition is true if (queue.size() == 0) { rle = new AvailabilityDataRLE(rle.getMeasurement(), rle.getEndtime(), next, AVAIL_UNKNOWN); queue.addLast(rle); } rle = (AvailabilityDataRLE) queue.getLast(); availStartime = rle.getStartime(); availEndtime = rle.getEndtime(); endtime = availEndtime; } HighLowMetricValue val; if (curr >= availStartime) { val = getMetricValue(queue, curr); } else if (prependUnknowns) { val = new HighLowMetricValue(AVAIL_UNKNOWN, curr); val.incrementCount(); } else { i++; continue; } if (rtn.size() <= i) { rtn.add(round(val)); } else { updateMetricValue(val, (HighLowMetricValue) rtn.get(i)); } i++; } } if (rtn.size() == 0) { rtn.addAll(getDefaultHistoricalAvail(end)); } return rtn; }
From source file:org.deeplearning4j.nn.graph.ComputationGraph.java
/** * Do backprop (gradient calculation)//from ww w. j av a2 s .c o m * * @param truncatedBPTT false: normal backprop. true: calculate gradients using truncated BPTT for RNN layers * @param externalEpsilons null usually (for typical supervised learning). If not null (and length > 0) then assume that * the user has provided some errors externally, as they would do for example in reinforcement * learning situations. */ protected void calcBackpropGradients(boolean truncatedBPTT, INDArray... externalEpsilons) { if (flattenedGradients == null) initGradientsView(); LinkedList<Triple<String, INDArray, Character>> gradients = new LinkedList<>(); //Do backprop according to the reverse of the topological ordering of the network boolean[] setVertexEpsilon = new boolean[topologicalOrder.length]; //If true: already set epsilon for this vertex; later epsilons should be *added* to the existing one, not set for (int i = topologicalOrder.length - 1; i >= 0; i--) { GraphVertex current = vertices[topologicalOrder[i]]; if (current.isInputVertex()) continue; //No op //FIXME: make the frozen vertex feature extraction more flexible if (current.hasLayer() && current.getLayer() instanceof FrozenLayer) break; if (current.isOutputVertex()) { //Two reasons for a vertex to be an output vertex: //(a) it's an output layer (i.e., instanceof IOutputLayer), or //(b) it's a normal layer, but it has been marked as an output layer for use in external errors - for reinforcement learning, for example int thisOutputNumber = configuration.getNetworkOutputs().indexOf(current.getVertexName()); if (current.getLayer() instanceof IOutputLayer) { IOutputLayer outputLayer = (IOutputLayer) current.getLayer(); INDArray currLabels = labels[thisOutputNumber]; outputLayer.setLabels(currLabels); } else { current.setEpsilon(externalEpsilons[thisOutputNumber]); setVertexEpsilon[topologicalOrder[i]] = true; } } Pair<Gradient, INDArray[]> pair = current.doBackward(truncatedBPTT); INDArray[] epsilons = pair.getSecond(); //Inputs to the current GraphVertex: VertexIndices[] inputVertices = current.getInputVertices(); //Set epsilons for the vertices that provide inputs to this vertex: if (inputVertices != null) { int j = 0; for (VertexIndices v : inputVertices) { GraphVertex gv = vertices[v.getVertexIndex()]; if (setVertexEpsilon[gv.getVertexIndex()]) { //This vertex: must output to multiple vertices... we want to add the epsilons here INDArray currentEps = gv.getEpsilon(); gv.setEpsilon(currentEps.add(epsilons[j++])); //TODO: in some circumstances, it may be safe to do in-place add (but not always) } else { gv.setEpsilon(epsilons[j++]); } setVertexEpsilon[gv.getVertexIndex()] = true; } } if (pair.getFirst() != null) { Gradient g = pair.getFirst(); Map<String, INDArray> map = g.gradientForVariable(); LinkedList<Triple<String, INDArray, Character>> tempList = new LinkedList<>(); for (Map.Entry<String, INDArray> entry : map.entrySet()) { String origName = entry.getKey(); String newName = current.getVertexName() + "_" + origName; tempList.addFirst( new Triple<>(newName, entry.getValue(), g.flatteningOrderForVariable(origName))); } for (Triple<String, INDArray, Character> t : tempList) gradients.addFirst(t); } } //Now, add the gradients in the order we need them in for flattening (same as params order) Gradient gradient = new DefaultGradient(flattenedGradients); for (Triple<String, INDArray, Character> t : gradients) { gradient.setGradientFor(t.getFirst(), t.getSecond(), t.getThird()); } this.gradient = gradient; }
From source file:org.jnap.core.mvc.async.AsyncRequestInterceptor.java
@Override public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { if (isAsync(request, handler)) { Method handlerMethod = (Method) request .getAttribute(RestfulHandlerAdapter.CURRENT_HANDLER_METHOD_ATTRIBUTE); if (handlerMethod != null) { LinkedList<AsyncResponseHandler> handlers = new LinkedList<AsyncResponseHandler>(); AsyncResponseHandler responseHandler = null; if (AsyncResponseModel.class.isInstance(modelAndView)) { // TODO AsyncState.SUSPEND_RESPONSE }/* w w w. java2s . c om*/ if (handlerMethod.isAnnotationPresent(Broadcast.class)) { Broadcast annotation = handlerMethod.getAnnotation(Broadcast.class); int delay = annotation.delay(); Class[] suspendTimeout = annotation.value(); AsyncState state = annotation.resumeOnBroadcast() ? AsyncState.RESUME_ON_BROADCAST : AsyncState.BROADCAST; responseHandler = new AsyncResponseHandler(state, delay, 0, SCOPE.APPLICATION, true, suspendTimeout, null); handlers.addLast(responseHandler); if (handlerMethod.isAnnotationPresent(Cluster.class)) { // TODO add @Cluster support } } if (handlerMethod.isAnnotationPresent(Suspend.class)) { Suspend annotation = handlerMethod.getAnnotation(Suspend.class); long suspendTimeout = annotation.period(); suspendTimeout = TimeUnitConverter.convert(suspendTimeout, annotation.timeUnit()); Suspend.SCOPE scope = annotation.scope(); boolean outputComments = annotation.outputComments(); boolean trackable = false; // TODO add Trackable support // if (TrackableResource.class.isAssignableFrom(am.getMethod().getReturnType())) { // trackable = true; // } AsyncState state = annotation.resumeOnBroadcast() ? AsyncState.SUSPEND_RESUME : AsyncState.SUSPEND; if (trackable) { state = AsyncState.SUSPEND_TRACKABLE; } responseHandler = new AsyncResponseHandler(state, suspendTimeout, 0, scope, outputComments); responseHandler.setListeners(createListeners(annotation.listeners())); handlers.addFirst(responseHandler); } if (handlerMethod.isAnnotationPresent(Subscribe.class)) { boolean trackable = false; // TODO add Trackable support // if (TrackableResource.class.isAssignableFrom(am.getMethod().getReturnType())) { // trackable = true; // } Subscribe annotation = handlerMethod.getAnnotation(Subscribe.class); AsyncState state = trackable ? AsyncState.SUBSCRIBE_TRACKABLE : AsyncState.SUBSCRIBE; String topic = annotation.value(); // TODO add SpEL support responseHandler = new AsyncResponseHandler(state, 30000, -1, Suspend.SCOPE.APPLICATION, false, null, topic); responseHandler.setListeners(createListeners(annotation.listeners())); handlers.addFirst(responseHandler); } if (handlerMethod.isAnnotationPresent(Publish.class)) { String topic = handlerMethod.getAnnotation(Publish.class).value(); // TODO add SpEL support responseHandler = new AsyncResponseHandler(AsyncState.PUBLISH, 30000, -1, Suspend.SCOPE.APPLICATION, false, null, topic); handlers.addFirst(responseHandler); } if (handlerMethod.isAnnotationPresent(Resume.class)) { handlers.addFirst(new AsyncResponseHandler(AsyncState.RESUME, handlerMethod.getAnnotation(Resume.class).value())); } if (handlerMethod.isAnnotationPresent(Schedule.class)) { Schedule annotation = handlerMethod.getAnnotation(Schedule.class); AsyncState state = annotation.resumeOnBroadcast() ? AsyncState.SCHEDULE_RESUME : AsyncState.SCHEDULE; handlers.addFirst(new AsyncResponseHandler(state, annotation.period(), annotation.waitFor())); } for (AsyncResponseHandler asyncHandler : handlers) { asyncHandler.handle(request, response, modelAndView); } } else { logger.warn("Atmosphere annotation support disabled on this request."); } } }
From source file:org.deeplearning4j.nn.multilayer.MultiLayerNetwork.java
/** Equivalent to backprop(), but calculates gradient for truncated BPTT instead. */ protected void truncatedBPTTGradient() { if (flattenedGradients == null) initGradientsView();/*from www.j a v a 2s .com*/ String multiGradientKey; gradient = new DefaultGradient(); Layer currLayer; if (!(getOutputLayer() instanceof IOutputLayer)) { log.warn( "Warning: final layer isn't output layer. You cannot use backprop (truncated BPTT) without an output layer."); return; } IOutputLayer outputLayer = (IOutputLayer) getOutputLayer(); if (labels == null) throw new IllegalStateException("No labels found"); if (outputLayer.conf().getLayer().getWeightInit() == WeightInit.ZERO) { throw new IllegalStateException( "Output layer weights cannot be initialized to zero when using backprop."); } outputLayer.setLabels(labels); //calculate and apply the backward gradient for every layer int numLayers = getnLayers(); //Store gradients is a list; used to ensure iteration order in DefaultGradient linked hash map. i.e., layer 0 first instead of output layer LinkedList<Pair<String, INDArray>> gradientList = new LinkedList<>(); Pair<Gradient, INDArray> currPair = outputLayer.backpropGradient(null); for (Map.Entry<String, INDArray> entry : currPair.getFirst().gradientForVariable().entrySet()) { multiGradientKey = String.valueOf(numLayers - 1) + "_" + entry.getKey(); gradientList.addLast(new Pair<>(multiGradientKey, entry.getValue())); } if (getLayerWiseConfigurations().getInputPreProcess(numLayers - 1) != null) currPair = new Pair<>(currPair.getFirst(), this.layerWiseConfigurations .getInputPreProcess(numLayers - 1).backprop(currPair.getSecond(), getInputMiniBatchSize())); // Calculate gradients for previous layers & drops output layer in count for (int j = numLayers - 2; j >= 0; j--) { currLayer = getLayer(j); if (currLayer instanceof RecurrentLayer) { currPair = ((RecurrentLayer) currLayer).tbpttBackpropGradient(currPair.getSecond(), layerWiseConfigurations.getTbpttBackLength()); } else { currPair = currLayer.backpropGradient(currPair.getSecond()); } LinkedList<Pair<String, INDArray>> tempList = new LinkedList<>(); for (Map.Entry<String, INDArray> entry : currPair.getFirst().gradientForVariable().entrySet()) { multiGradientKey = String.valueOf(j) + "_" + entry.getKey(); tempList.addFirst(new Pair<>(multiGradientKey, entry.getValue())); } for (Pair<String, INDArray> pair : tempList) gradientList.addFirst(pair); //Pass epsilon through input processor before passing to next layer (if applicable) if (getLayerWiseConfigurations().getInputPreProcess(j) != null) currPair = new Pair<>(currPair.getFirst(), getLayerWiseConfigurations().getInputPreProcess(j) .backprop(currPair.getSecond(), getInputMiniBatchSize())); } //Add gradients to Gradients, in correct order for (Pair<String, INDArray> pair : gradientList) gradient.setGradientFor(pair.getFirst(), pair.getSecond()); }
From source file:org.deeplearning4j.nn.multilayer.MultiLayerNetwork.java
/** Calculate gradients and errors. Used in two places: * (a) backprop (for standard multi layer network learning) * (b) backpropGradient (layer method, for when MultiLayerNetwork is used as a layer) * @param epsilon Errors (technically errors .* activations). Not used if withOutputLayer = true * @param withOutputLayer if true: assume last layer is output layer, and calculate errors based on labels. In this * case, the epsilon input is not used (may/should be null). * If false: calculate backprop gradients * @return Gradients and the error (epsilon) at the input */// w ww .ja va2 s. c om protected Pair<Gradient, INDArray> calcBackpropGradients(INDArray epsilon, boolean withOutputLayer) { if (flattenedGradients == null) initGradientsView(); String multiGradientKey; Gradient gradient = new DefaultGradient(flattenedGradients); Layer currLayer; //calculate and apply the backward gradient for every layer /** * Skip the output layer for the indexing and just loop backwards updating the coefficients for each layer. * (when withOutputLayer == true) * * Activate applies the activation function for each layer and sets that as the input for the following layer. * * Typical literature contains most trivial case for the error calculation: wT * weights * This interpretation transpose a few things to get mini batch because ND4J is rows vs columns organization for params */ int numLayers = getnLayers(); //Store gradients is a list; used to ensure iteration order in DefaultGradient linked hash map. i.e., layer 0 first instead of output layer LinkedList<Triple<String, INDArray, Character>> gradientList = new LinkedList<>(); int layerFrom; Pair<Gradient, INDArray> currPair; if (withOutputLayer) { if (!(getOutputLayer() instanceof IOutputLayer)) { log.warn( "Warning: final layer isn't output layer. You cannot use backprop without an output layer."); return null; } IOutputLayer outputLayer = (IOutputLayer) getOutputLayer(); if (labels == null) throw new IllegalStateException("No labels found"); outputLayer.setLabels(labels); currPair = outputLayer.backpropGradient(null); for (Map.Entry<String, INDArray> entry : currPair.getFirst().gradientForVariable().entrySet()) { String origName = entry.getKey(); multiGradientKey = String.valueOf(numLayers - 1) + "_" + origName; gradientList.addLast(new Triple<>(multiGradientKey, entry.getValue(), currPair.getFirst().flatteningOrderForVariable(origName))); } if (getLayerWiseConfigurations().getInputPreProcess(numLayers - 1) != null) currPair = new Pair<>(currPair.getFirst(), this.layerWiseConfigurations .getInputPreProcess(numLayers - 1).backprop(currPair.getSecond(), getInputMiniBatchSize())); layerFrom = numLayers - 2; } else { currPair = new Pair<>(null, epsilon); layerFrom = numLayers - 1; } // Calculate gradients for previous layers & drops output layer in count for (int j = layerFrom; j >= 0; j--) { currLayer = getLayer(j); if (currLayer instanceof FrozenLayer) break; currPair = currLayer.backpropGradient(currPair.getSecond()); LinkedList<Triple<String, INDArray, Character>> tempList = new LinkedList<>(); for (Map.Entry<String, INDArray> entry : currPair.getFirst().gradientForVariable().entrySet()) { String origName = entry.getKey(); multiGradientKey = String.valueOf(j) + "_" + origName; tempList.addFirst(new Triple<>(multiGradientKey, entry.getValue(), currPair.getFirst().flatteningOrderForVariable(origName))); } for (Triple<String, INDArray, Character> triple : tempList) gradientList.addFirst(triple); //Pass epsilon through input processor before passing to next layer (if applicable) if (getLayerWiseConfigurations().getInputPreProcess(j) != null) currPair = new Pair<>(currPair.getFirst(), getLayerWiseConfigurations().getInputPreProcess(j) .backprop(currPair.getSecond(), getInputMiniBatchSize())); } //Add gradients to Gradients (map), in correct order for (Triple<String, INDArray, Character> triple : gradientList) { gradient.setGradientFor(triple.getFirst(), triple.getSecond(), triple.getThird()); } return new Pair<>(gradient, currPair.getSecond()); }