List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService
public ExecutorCompletionService(Executor executor)
From source file:com.splout.db.hadoop.GeneratorCMD.java
public int run(String[] args) throws Exception { JCommander jComm = new JCommander(this); jComm.setProgramName(//from w w w .ja va 2 s . co m "Splout Tablespaces Generator. Generates tablespaces, ready to be deployed to a Splout Cluster."); try { jComm.parse(args); } catch (Throwable t) { t.printStackTrace(); jComm.usage(); return -1; } if (parallelism < 1) { System.err.println("Parallelism must be greater than 0."); System.exit(1); } log.info("Parsing input parameters..."); // All the tablespaces that will be generated and deployed atomically, hashed by their name // We generate this first so we can detect errors in the configuration before even using Hadoop Map<String, TablespaceSpec> tablespacesToGenerate = new HashMap<String, TablespaceSpec>(); // Partition maps to reuse at indexation. Used when sampling is skipped. final Map<String, PartitionMap> partitionMapsToReuse = new HashMap<String, PartitionMap>(); for (String tablespaceFile : tablespaceFiles) { Path file = new Path(tablespaceFile); FileSystem fS = FileSystem.get(file.toUri(), getConf()); if (!fS.exists(file)) { throw new IllegalArgumentException("Config input file: " + file + " doesn't exist!"); } String strContents = HadoopUtils.fileToString(fS, file); JSONTablespaceDefinition def = JSONSerDe.deSer(strContents, JSONTablespaceDefinition.class); TablespaceSpec spec = def.build(conf); String name = def.getName(); tablespacesToGenerate.put(name, spec); // Reusing partition maps? if (qnodeURL != null) { partitionMapsToReuse.put(name, retrievePartitionMapfromQNode(name)); } } if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) { File nativeLibs = new File("native"); if (nativeLibs.exists()) { SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf); } } Path out = new Path(output); FileSystem outFs = out.getFileSystem(getConf()); HadoopUtils.deleteIfExists(outFs, out); ExecutorService executor = Executors.newFixedThreadPool(parallelism); ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(executor); ArrayList<Future<Boolean>> generatorFutures = new ArrayList<Future<Boolean>>(); // Generate each tablespace for (final Map.Entry<String, TablespaceSpec> tablespace : tablespacesToGenerate.entrySet()) { Path tablespaceOut = new Path(out, tablespace.getKey()); TablespaceSpec spec = tablespace.getValue(); log.info("Generating view with Hadoop (" + tablespace.getKey() + ")"); final TablespaceGenerator viewGenerator = new TablespaceGenerator(spec, tablespaceOut, this.getClass()); generatorFutures.add(ecs.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { if (qnodeURL == null) { viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions()); return true; } else { viewGenerator.generateView(conf, partitionMapsToReuse.get(tablespace.getKey())); return true; } } })); } // Waiting all tasks to finish. for (int i = 0; i < tablespacesToGenerate.size(); i++) { // Get will throw an exception if the callable returned it. try { ecs.take().get(); } catch (ExecutionException e) { // One job was wrong. Stopping the rest. for (Future<Boolean> task : generatorFutures) { task.cancel(true); } executor.shutdown(); throw e; } } executor.shutdown(); log.info("Done!"); return 0; }
From source file:org.apache.flume.channel.kafka.TestKafkaChannel.java
/** * This method starts a channel, puts events into it. The channel is then * stopped and restarted. Then we check to make sure if all events we put * come out. Optionally, 10 events are rolled back, * and optionally we restart the agent immediately after and we try to pull it * out.//from w ww.j a va 2 s . c o m * * @param rollback * @param retryAfterRollback * @throws Exception */ private void doTestStopAndStart(boolean rollback, boolean retryAfterRollback) throws Exception { final KafkaChannel channel = startChannel(true); ExecutorService underlying = Executors.newCachedThreadPool(); ExecutorCompletionService<Void> submitterSvc = new ExecutorCompletionService<Void>(underlying); final List<List<Event>> events = createBaseList(); putEvents(channel, events, submitterSvc); int completed = 0; wait(submitterSvc, 5); channel.stop(); final KafkaChannel channel2 = startChannel(true); int total = 50; if (rollback && !retryAfterRollback) { total = 40; } final List<Event> eventsPulled = pullEvents(channel2, submitterSvc, total, rollback, retryAfterRollback); wait(submitterSvc, 5); channel2.stop(); if (!retryAfterRollback && rollback) { final KafkaChannel channel3 = startChannel(true); int expectedRemaining = 50 - eventsPulled.size(); final List<Event> eventsPulled2 = pullEvents(channel3, submitterSvc, expectedRemaining, false, false); wait(submitterSvc, 5); Assert.assertEquals(expectedRemaining, eventsPulled2.size()); eventsPulled.addAll(eventsPulled2); channel3.stop(); } underlying.shutdownNow(); verify(eventsPulled); }
From source file:com.opengamma.integration.viewer.status.impl.ViewStatusCalculationWorker.java
public ViewStatusResultAggregator run() { ViewStatusResultAggregator aggregator = new ViewStatusResultAggregatorImpl(); CompletionService<PerViewStatusResult> completionService = new ExecutorCompletionService<PerViewStatusResult>( _executor);//ww w . jav a 2s. c om //submit task to executor to run partitioned by security type for (String securityType : _valueRequirementBySecType.keySet()) { Collection<String> valueRequirements = _valueRequirementBySecType.get(securityType); completionService.submit(new ViewStatusCalculationTask(_toolContext, _portfolioId, _user, securityType, valueRequirements, _marketDataSpecification)); } try { // process all completed task for (int i = 0; i < _valueRequirementBySecType.size(); i++) { Future<PerViewStatusResult> futureTask = completionService.take(); PerViewStatusResult perViewStatusResult = futureTask.get(); for (ViewStatusKey viewStatusKey : perViewStatusResult.keySet()) { aggregator.putStatus(viewStatusKey, perViewStatusResult.get(viewStatusKey)); } } } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } catch (ExecutionException ex) { throw new OpenGammaRuntimeException("Error running View status report", ex.getCause()); } return aggregator; }
From source file:pl.edu.icm.cermine.libsvm.parameters.SVMParameterFinder.java
public void run(String inputFile, String ext, int threads, int kernel, int degree, int minc, int maxc, int ming, int maxg) throws AnalysisException, InterruptedException, ExecutionException { List<TrainingSample<BxZoneLabel>> samples = getSamples(inputFile, ext); ExecutorService executor = Executors.newFixedThreadPool(3); CompletionService<EvaluationParams> completionService = new ExecutorCompletionService<EvaluationParams>( executor);/*from w w w .jav a 2 s . com*/ double bestRate = 0; int bestclog = 0; int bestglog = 0; int submitted = 0; for (int clog = minc; clog <= maxc; clog++) { for (int glog = maxg; glog >= ming; glog--) { completionService.submit(new Evaluator(samples, new EvaluationParams(clog, glog), kernel, degree)); submitted++; } } while (submitted > 0) { Future<EvaluationParams> f1 = completionService.take(); EvaluationParams p = f1.get(); if (p.rate > bestRate) { bestRate = p.rate; bestclog = p.clog; bestglog = p.glog; } System.out.println("Gamma: " + p.glog + ", C: " + p.clog + ", rate: " + p.rate + " (Best: " + bestglog + " " + bestclog + " " + bestRate + ")"); submitted--; } executor.shutdown(); }
From source file:org.commonjava.maven.galley.internal.TransferManagerImpl.java
@PostConstruct public void init() { batchExecutor = new ExecutorCompletionService<>(executorService); }
From source file:org.nickelproject.nickel.blobStore.S3BlobStore.java
private void putMultiPartByteArray(final BlobRef blobRef, final byte[] pBytes) { final CompletionService<PartETag> completionService = new ExecutorCompletionService<PartETag>(executor); final String key = blobRef.toString(); final PartTransferInterface partGetter = RetryProxy.newInstance(PartTransferInterface.class, new PartTransfer()); final String uploadId = partGetter.initiateUpload(key); int partNumber = 1; for (int start = 0; start < pBytes.length; start += uploadPartSize) { final long length = Math.min(uploadPartSize, pBytes.length - start); completionService.submit(new PutPartCallable(start, (int) length, pBytes, partNumber++, key, uploadId, start + uploadPartSize >= pBytes.length)); }/* w w w .ja v a 2 s . c o m*/ final List<PartETag> partETags = Lists.newArrayList(); for (int i = 1; i < partNumber; i++) { try { partETags.add(completionService.take().get()); } catch (final InterruptedException e1) { Thread.currentThread().interrupt(); throw RethrownException.rethrow(e1); } catch (final ExecutionException e2) { throw RethrownException.rethrow(e2); } } partGetter.completeUpload(key, uploadId, partETags); }
From source file:com.netflix.curator.framework.recipes.leader.TestLeaderLatch.java
@Test public void testWaiting() throws Exception { final int PARTICIPANT_QTY = 10; ExecutorService executorService = Executors.newFixedThreadPool(PARTICIPANT_QTY); ExecutorCompletionService<Void> service = new ExecutorCompletionService<Void>(executorService); final Timing timing = new Timing(); final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new RetryOneTime(1)); try {/*from ww w . j a va 2 s.co m*/ client.start(); final AtomicBoolean thereIsALeader = new AtomicBoolean(false); for (int i = 0; i < PARTICIPANT_QTY; ++i) { service.submit(new Callable<Void>() { @Override public Void call() throws Exception { LeaderLatch latch = new LeaderLatch(client, PATH_NAME); try { latch.start(); Assert.assertTrue(latch.await(timing.forWaiting().seconds(), TimeUnit.SECONDS)); Assert.assertTrue(thereIsALeader.compareAndSet(false, true)); Thread.sleep((int) (10 * Math.random())); } finally { thereIsALeader.set(false); latch.close(); } return null; } }); } for (int i = 0; i < PARTICIPANT_QTY; ++i) { service.take().get(); } } finally { executorService.shutdown(); IOUtils.closeQuietly(client); } }
From source file:org.tinymediamanager.scraper.imdb.ImdbMetadataProvider.java
@Override public MediaMetadata getMetadata(MediaScrapeOptions options) throws Exception { LOGGER.debug("getMetadata() " + options.toString()); // check if there is a md in the result if (options.getResult() != null && options.getResult().getMetadata() != null) { LOGGER.debug("IMDB: getMetadata from cache: " + options.getResult()); return options.getResult().getMetadata(); }/*from w w w. j a v a 2 s.c o m*/ MediaMetadata md = new MediaMetadata(providerInfo.getId()); String imdbId = ""; // imdbId from searchResult if (options.getResult() != null) { imdbId = options.getResult().getIMDBId(); } // imdbid from scraper option if (!MetadataUtil.isValidImdbId(imdbId)) { imdbId = options.getImdbId(); } if (!MetadataUtil.isValidImdbId(imdbId)) { return md; } LOGGER.debug("IMDB: getMetadata(imdbId): " + imdbId); md.setId(MediaMetadata.IMDBID, imdbId); ExecutorCompletionService<Document> compSvcImdb = new ExecutorCompletionService<Document>(executor); ExecutorCompletionService<MediaMetadata> compSvcTmdb = new ExecutorCompletionService<MediaMetadata>( executor); // worker for imdb request (/combined) (everytime from akas.imdb.com) // StringBuilder sb = new StringBuilder(imdbSite.getSite()); StringBuilder sb = new StringBuilder(ImdbSiteDefinition.IMDB_COM.getSite()); sb.append("title/"); sb.append(imdbId); sb.append("/combined"); Callable<Document> worker = new ImdbWorker(sb.toString(), options.getLanguage().name(), options.getCountry().getAlpha2()); Future<Document> futureCombined = compSvcImdb.submit(worker); // worker for imdb request (/plotsummary) (from chosen site) Future<Document> futurePlotsummary = null; sb = new StringBuilder(imdbSite.getSite()); sb.append("title/"); sb.append(imdbId); sb.append("/plotsummary"); worker = new ImdbWorker(sb.toString(), options.getLanguage().name(), options.getCountry().getAlpha2()); futurePlotsummary = compSvcImdb.submit(worker); // worker for tmdb request Future<MediaMetadata> futureTmdb = null; if (options.isScrapeImdbForeignLanguage() || options.isScrapeCollectionInfo()) { Callable<MediaMetadata> worker2 = new TmdbWorker(imdbId, options.getLanguage(), options.getCountry()); futureTmdb = compSvcTmdb.submit(worker2); } Document doc; doc = futureCombined.get(); /* * title and year have the following structure * * <div id="tn15title"><h1>Merida - Legende der Highlands <span>(<a href="/year/2012/">2012</a>) <span class="pro-link">...</span> <span * class="title-extra">Brave <i>(original title)</i></span> </span></h1> </div> */ // parse title and year Element title = doc.getElementById("tn15title"); if (title != null) { Element element = null; // title Elements elements = title.getElementsByTag("h1"); if (elements.size() > 0) { element = elements.first(); String movieTitle = cleanString(element.ownText()); md.storeMetadata(MediaMetadata.TITLE, movieTitle); } // year elements = title.getElementsByTag("span"); if (elements.size() > 0) { element = elements.first(); String content = element.text(); // search year Pattern yearPattern = Pattern.compile("\\(([0-9]{4})|/\\)"); Matcher matcher = yearPattern.matcher(content); while (matcher.find()) { if (matcher.group(1) != null) { String movieYear = matcher.group(1); md.storeMetadata(MediaMetadata.YEAR, movieYear); break; } } } // original title elements = title.getElementsByAttributeValue("class", "title-extra"); if (elements.size() > 0) { element = elements.first(); String content = element.text(); content = content.replaceAll("\\(original title\\)", "").trim(); md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, content); } } // poster Element poster = doc.getElementById("primary-poster"); if (poster != null) { String posterUrl = poster.attr("src"); posterUrl = posterUrl.replaceAll("SX[0-9]{2,4}_", "SX400_"); posterUrl = posterUrl.replaceAll("SY[0-9]{2,4}_", "SY400_"); processMediaArt(md, MediaArtworkType.POSTER, "Poster", posterUrl); } /* * <div class="starbar-meta"> <b>7.4/10</b> <a href="ratings" class="tn15more">52,871 votes</a> » </div> */ // rating and rating count Element ratingElement = doc.getElementById("tn15rating"); if (ratingElement != null) { Elements elements = ratingElement.getElementsByClass("starbar-meta"); if (elements.size() > 0) { Element div = elements.get(0); // rating comes in <b> tag Elements b = div.getElementsByTag("b"); if (b.size() == 1) { String ratingAsString = b.text(); Pattern ratingPattern = Pattern.compile("([0-9]\\.[0-9])/10"); Matcher matcher = ratingPattern.matcher(ratingAsString); while (matcher.find()) { if (matcher.group(1) != null) { float rating = 0; try { rating = Float.valueOf(matcher.group(1)); } catch (Exception e) { } md.storeMetadata(MediaMetadata.RATING, rating); break; } } } // count Elements a = div.getElementsByAttributeValue("href", "ratings"); if (a.size() == 1) { String countAsString = a.text().replaceAll("[.,]|votes", "").trim(); int voteCount = 0; try { voteCount = Integer.parseInt(countAsString); } catch (Exception e) { } md.storeMetadata(MediaMetadata.VOTE_COUNT, voteCount); } } // top250 elements = ratingElement.getElementsByClass("starbar-special"); if (elements.size() > 0) { Elements a = elements.get(0).getElementsByTag("a"); if (a.size() > 0) { Element anchor = a.get(0); Pattern topPattern = Pattern.compile("Top 250: #([0-9]{1,3})"); Matcher matcher = topPattern.matcher(anchor.ownText()); while (matcher.find()) { if (matcher.group(1) != null) { int top250 = 0; try { top250 = Integer.parseInt(matcher.group(1)); } catch (Exception e) { } md.storeMetadata(MediaMetadata.TOP_250, top250); } } } } } // parse all items coming by <div class="info"> Elements elements = doc.getElementsByClass("info"); for (Element element : elements) { // only parse divs if (!"div".equals(element.tag().getName())) { continue; } // elements with h5 are the titles of the values Elements h5 = element.getElementsByTag("h5"); if (h5.size() > 0) { Element firstH5 = h5.first(); String h5Title = firstH5.text(); // release date /* * <div class="info"><h5>Release Date:</h5><div class="info-content">5 January 1996 (USA)<a class="tn15more inline" * href="/title/tt0114746/releaseinfo" * onclick="(new Image()).src='/rg/title-tease/releasedates/images/b.gif?link=/title/tt0114746/releaseinfo';"> See more</a> </div></div> */ if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getReleaseDate() + ".*")) { Elements div = element.getElementsByClass("info-content"); if (div.size() > 0) { Element releaseDateElement = div.first(); String releaseDate = cleanString(releaseDateElement.ownText().replaceAll("", "")); Pattern pattern = Pattern.compile("(.*)\\(.*\\)"); Matcher matcher = pattern.matcher(releaseDate); if (matcher.find()) { try { SimpleDateFormat sdf = new SimpleDateFormat("d MMM yyyy"); Date parsedDate = sdf.parse(matcher.group(1)); sdf = new SimpleDateFormat("dd-MM-yyyy"); md.storeMetadata(MediaMetadata.RELEASE_DATE, sdf.format(parsedDate)); } catch (Exception e) { } } } } /* * <div class="info"><h5>Tagline:</h5><div class="info-content"> (7) To Defend Us... <a class="tn15more inline" * href="/title/tt0472033/taglines" onClick= "(new Image()).src='/rg/title-tease/taglines/images/b.gif?link=/title/tt0472033/taglines';" >See * more</a> » </div></div> */ // tagline if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getTagline() + ".*") && !options.isScrapeImdbForeignLanguage()) { Elements div = element.getElementsByClass("info-content"); if (div.size() > 0) { Element taglineElement = div.first(); String tagline = cleanString(taglineElement.ownText().replaceAll("", "")); md.storeMetadata(MediaMetadata.TAGLINE, tagline); } } /* * <div class="info-content"><a href="/Sections/Genres/Animation/">Animation</a> | <a href="/Sections/Genres/Action/">Action</a> | <a * href="/Sections/Genres/Adventure/">Adventure</a> | <a href="/Sections/Genres/Fantasy/">Fantasy</a> | <a * href="/Sections/Genres/Mystery/">Mystery</a> | <a href="/Sections/Genres/Sci-Fi/">Sci-Fi</a> | <a * href="/Sections/Genres/Thriller/">Thriller</a> <a class="tn15more inline" href="/title/tt0472033/keywords" onClick= * "(new Image()).src='/rg/title-tease/keywords/images/b.gif?link=/title/tt0472033/keywords';" > See more</a> » </div> */ // genres are only scraped from akas.imdb.com if (h5Title.matches("(?i)" + imdbSite.getGenre() + "(.*)")) { Elements div = element.getElementsByClass("info-content"); if (div.size() > 0) { Elements a = div.first().getElementsByTag("a"); for (Element anchor : a) { if (anchor.attr("href").matches("/Sections/Genres/.*")) { md.addGenre(getTmmGenre(anchor.ownText())); } } } } // } /* * <div class="info"><h5>Runtime:</h5><div class="info-content">162 min | 171 min (special edition) | 178 min (extended cut)</div></div> */ // runtime // if (h5Title.matches("(?i)" + imdbSite.getRuntime() + ".*")) { if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getRuntime() + ".*")) { Elements div = element.getElementsByClass("info-content"); if (div.size() > 0) { Element taglineElement = div.first(); String first = taglineElement.ownText().split("\\|")[0]; String runtimeAsString = cleanString(first.replaceAll("min", "")); int runtime = 0; try { runtime = Integer.parseInt(runtimeAsString); } catch (Exception e) { // try to filter out the first number we find Pattern runtimePattern = Pattern.compile("([0-9]{2,3})"); Matcher matcher = runtimePattern.matcher(runtimeAsString); if (matcher.find()) { runtime = Integer.parseInt(matcher.group(0)); } } md.storeMetadata(MediaMetadata.RUNTIME, runtime); } } /* * <div class="info"><h5>Country:</h5><div class="info-content"><a href="/country/fr">France</a> | <a href="/country/es">Spain</a> | <a * href="/country/it">Italy</a> | <a href="/country/hu">Hungary</a></div></div> */ // country if (h5Title.matches("(?i)Country.*")) { Elements a = element.getElementsByTag("a"); String countries = ""; for (Element anchor : a) { Pattern pattern = Pattern.compile("/country/(.*)"); Matcher matcher = pattern.matcher(anchor.attr("href")); if (matcher.matches()) { String country = matcher.group(1); if (StringUtils.isNotEmpty(countries)) { countries += ", "; } countries += country.toUpperCase(); } } md.storeMetadata(MediaMetadata.COUNTRY, countries); } /* * <div class="info"><h5>Language:</h5><div class="info-content"><a href="/language/en">English</a> | <a href="/language/de">German</a> | <a * href="/language/fr">French</a> | <a href="/language/it">Italian</a></div> */ // Spoken languages if (h5Title.matches("(?i)Language.*")) { Elements a = element.getElementsByTag("a"); String spokenLanguages = ""; for (Element anchor : a) { Pattern pattern = Pattern.compile("/language/(.*)"); Matcher matcher = pattern.matcher(anchor.attr("href")); if (matcher.matches()) { String langu = matcher.group(1); if (StringUtils.isNotEmpty(spokenLanguages)) { spokenLanguages += ", "; } spokenLanguages += langu; } } md.storeMetadata(MediaMetadata.SPOKEN_LANGUAGES, spokenLanguages); } /* * <div class="info"><h5>Certification:</h5><div class="info-content"><a href="/search/title?certificates=us:pg">USA:PG</a> <i>(certificate * #47489)</i> | <a href="/search/title?certificates=ca:pg">Canada:PG</a> <i>(Ontario)</i> | <a * href="/search/title?certificates=au:pg">Australia:PG</a> | <a href="/search/title?certificates=in:u">India:U</a> | <a * href="/search/title?certificates=ie:pg">Ireland:PG</a> ...</div></div> */ // certification // if (h5Title.matches("(?i)" + imdbSite.getCertification() + ".*")) { if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getCertification() + ".*")) { Elements a = element.getElementsByTag("a"); for (Element anchor : a) { // certification for the right country if (anchor.attr("href").matches( "(?i)/search/title\\?certificates=" + options.getCountry().getAlpha2() + ".*")) { Pattern certificationPattern = Pattern.compile(".*:(.*)"); Matcher matcher = certificationPattern.matcher(anchor.ownText()); Certification certification = null; while (matcher.find()) { if (matcher.group(1) != null) { certification = Certification.getCertification(options.getCountry(), matcher.group(1)); } } if (certification != null) { md.addCertification(certification); break; } } } } } /* * <div id="director-info" class="info"> <h5>Director:</h5> <div class="info-content"><a href="/name/nm0000416/" onclick= * "(new Image()).src='/rg/directorlist/position-1/images/b.gif?link=name/nm0000416/';" >Terry Gilliam</a><br/> </div> </div> */ // director if ("director-info".equals(element.id())) { Elements a = element.getElementsByTag("a"); for (Element anchor : a) { if (anchor.attr("href").matches("/name/nm.*")) { MediaCastMember cm = new MediaCastMember(CastType.DIRECTOR); cm.setName(anchor.ownText()); md.addCastMember(cm); } } } } /* * <table class="cast"> <tr class="odd"><td class="hs"><a href="http://pro.imdb.com/widget/resume_redirect/" onClick= * "(new Image()).src='/rg/resume/prosystem/images/b.gif?link=http://pro.imdb.com/widget/resume_redirect/';" ><img src= * "http://i.media-imdb.com/images/SF9113d6f5b7cb1533c35313ccd181a6b1/tn15/no_photo.png" width="25" height="31" border="0"></td><td class="nm"><a * href="/name/nm0577828/" onclick= "(new Image()).src='/rg/castlist/position-1/images/b.gif?link=/name/nm0577828/';" >Joseph Melito</a></td><td * class="ddd"> ... </td><td class="char"><a href="/character/ch0003139/">Young Cole</a></td></tr> <tr class="even"><td class="hs"><a * href="/name/nm0000246/" onClick= "(new Image()).src='/rg/title-tease/tinyhead/images/b.gif?link=/name/nm0000246/';" ><img src= * "http://ia.media-imdb.com/images/M/MV5BMjA0MjMzMTE5OF5BMl5BanBnXkFtZTcwMzQ2ODE3Mw@@._V1._SY30_SX23_.jpg" width="23" height="32" * border="0"></a><br></td><td class="nm"><a href="/name/nm0000246/" onclick= * "(new Image()).src='/rg/castlist/position-2/images/b.gif?link=/name/nm0000246/';" >Bruce Willis</a></td><td class="ddd"> ... </td><td * class="char"><a href="/character/ch0003139/">James Cole</a></td></tr> <tr class="odd"><td class="hs"><a href="/name/nm0781218/" onClick= * "(new Image()).src='/rg/title-tease/tinyhead/images/b.gif?link=/name/nm0781218/';" ><img src= * "http://ia.media-imdb.com/images/M/MV5BODI1MTA2MjkxM15BMl5BanBnXkFtZTcwMTcwMDg2Nw@@._V1._SY30_SX23_.jpg" width="23" height="32" * border="0"></a><br></td><td class="nm"><a href="/name/nm0781218/" onclick= * "(new Image()).src='/rg/castlist/position-3/images/b.gif?link=/name/nm0781218/';" >Jon Seda</a></td><td class="ddd"> ... </td><td * class="char"><a href="/character/ch0003143/">Jose</a></td></tr>...</table> */ // cast elements = doc.getElementsByClass("cast"); if (elements.size() > 0) { Elements tr = elements.get(0).getElementsByTag("tr"); for (Element row : tr) { Elements td = row.getElementsByTag("td"); MediaCastMember cm = new MediaCastMember(); for (Element column : td) { // actor thumb if (column.hasClass("hs")) { Elements img = column.getElementsByTag("img"); if (img.size() > 0) { String thumbUrl = img.get(0).attr("src"); if (thumbUrl.contains("no_photo.png")) { cm.setImageUrl(""); } else { thumbUrl = thumbUrl.replaceAll("SX[0-9]{2,4}_", "SX400_"); thumbUrl = thumbUrl.replaceAll("SY[0-9]{2,4}_", ""); cm.setImageUrl(thumbUrl); } } } // actor name if (column.hasClass("nm")) { cm.setName(cleanString(column.text())); } // character if (column.hasClass("char")) { cm.setCharacter(cleanString(column.text())); } } if (StringUtils.isNotEmpty(cm.getName()) && StringUtils.isNotEmpty(cm.getCharacter())) { cm.setType(CastType.ACTOR); md.addCastMember(cm); } } } Element content = doc.getElementById("tn15content"); if (content != null) { elements = content.getElementsByTag("table"); for (Element table : elements) { // writers if (table.text().contains(ImdbSiteDefinition.IMDB_COM.getWriter())) { Elements anchors = table.getElementsByTag("a"); for (Element anchor : anchors) { if (anchor.attr("href").matches("/name/nm.*")) { MediaCastMember cm = new MediaCastMember(CastType.WRITER); cm.setName(anchor.ownText()); md.addCastMember(cm); } } } // producers if (table.text().contains(ImdbSiteDefinition.IMDB_COM.getProducers())) { Elements rows = table.getElementsByTag("tr"); for (Element row : rows) { if (row.text().contains(ImdbSiteDefinition.IMDB_COM.getProducers())) { continue; } Elements columns = row.children(); if (columns.size() == 0) { continue; } MediaCastMember cm = new MediaCastMember(CastType.PRODUCER); String name = cleanString(columns.get(0).text()); if (StringUtils.isBlank(name)) { continue; } cm.setName(name); if (columns.size() >= 3) { cm.setPart(cleanString(columns.get(2).text())); } md.addCastMember(cm); } } } } // Production companies elements = doc.getElementsByClass("blackcatheader"); for (Element blackcatheader : elements) { if (blackcatheader.ownText().equals(ImdbSiteDefinition.IMDB_COM.getProductionCompanies())) { Elements a = blackcatheader.nextElementSibling().getElementsByTag("a"); StringBuilder productionCompanies = new StringBuilder(); for (Element anchor : a) { if (StringUtils.isNotEmpty(productionCompanies)) { productionCompanies.append(", "); } productionCompanies.append(anchor.ownText()); } md.storeMetadata(MediaMetadata.PRODUCTION_COMPANY, productionCompanies.toString()); break; } } /* * plot from /plotsummary */ // build the url doc = null; doc = futurePlotsummary.get(); // imdb.com has another site structure if (imdbSite == ImdbSiteDefinition.IMDB_COM) { Elements zebraList = doc.getElementsByClass("zebraList"); if (zebraList != null && !zebraList.isEmpty()) { Elements odd = zebraList.get(0).getElementsByClass("odd"); if (odd.isEmpty()) { odd = zebraList.get(0).getElementsByClass("even"); // sometimes imdb has even } if (odd.size() > 0) { Elements p = odd.get(0).getElementsByTag("p"); if (p.size() > 0) { String plot = cleanString(p.get(0).ownText()); md.storeMetadata(MediaMetadata.PLOT, plot); } } } } else { Element wiki = doc.getElementById("swiki.2.1"); if (wiki != null) { String plot = cleanString(wiki.ownText()); md.storeMetadata(MediaMetadata.PLOT, plot); } } // title also from chosen site if we are not scraping akas.imdb.com if (imdbSite != ImdbSiteDefinition.IMDB_COM) { title = doc.getElementById("tn15title"); if (title != null) { Element element = null; // title elements = title.getElementsByClass("main"); if (elements.size() > 0) { element = elements.first(); String movieTitle = cleanString(element.ownText()); md.storeMetadata(MediaMetadata.TITLE, movieTitle); } } } // } // get data from tmdb? if (options.isScrapeImdbForeignLanguage() || options.isScrapeCollectionInfo()) { MediaMetadata tmdbMd = futureTmdb.get(); if (options.isScrapeImdbForeignLanguage() && tmdbMd != null && StringUtils.isNotBlank(tmdbMd.getStringValue(MediaMetadata.PLOT))) { // tmdbid md.setId(MediaMetadata.TMDBID, tmdbMd.getId(MediaMetadata.TMDBID)); // title md.storeMetadata(MediaMetadata.TITLE, tmdbMd.getStringValue(MediaMetadata.TITLE)); // original title md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, tmdbMd.getStringValue(MediaMetadata.ORIGINAL_TITLE)); // tagline md.storeMetadata(MediaMetadata.TAGLINE, tmdbMd.getStringValue(MediaMetadata.TAGLINE)); // plot md.storeMetadata(MediaMetadata.PLOT, tmdbMd.getStringValue(MediaMetadata.PLOT)); // collection info md.storeMetadata(MediaMetadata.COLLECTION_NAME, tmdbMd.getStringValue(MediaMetadata.COLLECTION_NAME)); md.storeMetadata(MediaMetadata.TMDBID_SET, tmdbMd.getIntegerValue(MediaMetadata.TMDBID_SET)); } if (options.isScrapeCollectionInfo() && tmdbMd != null) { md.storeMetadata(MediaMetadata.TMDBID_SET, tmdbMd.getIntegerValue(MediaMetadata.TMDBID_SET)); md.storeMetadata(MediaMetadata.COLLECTION_NAME, tmdbMd.getStringValue(MediaMetadata.COLLECTION_NAME)); } } // if we have still no original title, take the title if (StringUtils.isBlank(md.getStringValue(MediaMetadata.ORIGINAL_TITLE))) { md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, md.getStringValue(MediaMetadata.TITLE)); } return md; }
From source file:io.prestosql.plugin.accumulo.index.ColumnCardinalityCache.java
/** * Gets the cardinality for each {@link AccumuloColumnConstraint}. * Given constraints are expected to be indexed! Who knows what would happen if they weren't! * * @param schema Schema name/* w ww. jav a 2 s . c o m*/ * @param table Table name * @param auths Scan authorizations * @param idxConstraintRangePairs Mapping of all ranges for a given constraint * @param earlyReturnThreshold Smallest acceptable cardinality to return early while other tasks complete * @param pollingDuration Duration for polling the cardinality completion service * @return An immutable multimap of cardinality to column constraint, sorted by cardinality from smallest to largest * @throws TableNotFoundException If the metrics table does not exist * @throws ExecutionException If another error occurs; I really don't even know anymore. */ public Multimap<Long, AccumuloColumnConstraint> getCardinalities(String schema, String table, Authorizations auths, Multimap<AccumuloColumnConstraint, Range> idxConstraintRangePairs, long earlyReturnThreshold, Duration pollingDuration) { // Submit tasks to the executor to fetch column cardinality, adding it to the Guava cache if necessary CompletionService<Pair<Long, AccumuloColumnConstraint>> executor = new ExecutorCompletionService<>( executorService); idxConstraintRangePairs.asMap().forEach((key, value) -> executor.submit(() -> { long cardinality = getColumnCardinality(schema, table, auths, key.getFamily(), key.getQualifier(), value); LOG.debug("Cardinality for column %s is %s", key.getName(), cardinality); return Pair.of(cardinality, key); })); // Create a multi map sorted by cardinality ListMultimap<Long, AccumuloColumnConstraint> cardinalityToConstraints = MultimapBuilder.treeKeys() .arrayListValues().build(); try { boolean earlyReturn = false; int numTasks = idxConstraintRangePairs.asMap().entrySet().size(); do { // Sleep for the polling duration to allow concurrent tasks to run for this time Thread.sleep(pollingDuration.toMillis()); // Poll each task, retrieving the result if it is done for (int i = 0; i < numTasks; ++i) { Future<Pair<Long, AccumuloColumnConstraint>> futureCardinality = executor.poll(); if (futureCardinality != null && futureCardinality.isDone()) { Pair<Long, AccumuloColumnConstraint> columnCardinality = futureCardinality.get(); cardinalityToConstraints.put(columnCardinality.getLeft(), columnCardinality.getRight()); } } // If the smallest cardinality is present and below the threshold, set the earlyReturn flag Optional<Entry<Long, AccumuloColumnConstraint>> smallestCardinality = cardinalityToConstraints .entries().stream().findFirst(); if (smallestCardinality.isPresent()) { if (smallestCardinality.get().getKey() <= earlyReturnThreshold) { LOG.info("Cardinality %s, is below threshold. Returning early while other tasks finish", smallestCardinality); earlyReturn = true; } } } while (!earlyReturn && cardinalityToConstraints.entries().size() < numTasks); } catch (ExecutionException | InterruptedException e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting cardinality", e); } // Create a copy of the cardinalities return ImmutableMultimap.copyOf(cardinalityToConstraints); }
From source file:com.clustercontrol.http.factory.RunMonitorHttpScenario.java
/** * []????/*from w w w . j av a2s . c o m*/ * * ??1??ID??????ID?????????? * ????????runMonitorInfo??? * */ @Override protected boolean runMonitorInfo() throws FacilityNotFound, MonitorNotFound, EntityExistsException, InvalidRole, HinemosUnknown { m_log.debug("runMonitorInfo()"); m_now = new Date(HinemosTime.currentTimeMillis()); m_priorityMap = new HashMap<Integer, ArrayList<String>>(); m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_INFO), new ArrayList<String>()); m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_WARNING), new ArrayList<String>()); m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_CRITICAL), new ArrayList<String>()); m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_UNKNOWN), new ArrayList<String>()); try { // boolean run = this.setMonitorInfo(m_monitorTypeId, m_monitorId); if (!run) { // ? return true; } // setJudgementInfo(); // ?? setCheckInfo(); ArrayList<String> facilityList = null; ExecutorCompletionService<ArrayList<MonitorRunResultInfo>> ecs = new ExecutorCompletionService<ArrayList<MonitorRunResultInfo>>( ParallelExecution.instance().getExecutorService()); int taskCount = 0; if (!m_isMonitorJob) { // ?? // ID????? // /?true?????ID?? facilityList = new RepositoryControllerBean().getExecTargetFacilityIdList(m_facilityId, m_monitor.getOwnerRoleId()); if (facilityList.size() == 0) { return true; } m_isNode = new RepositoryControllerBean().isNode(m_facilityId); // ??????? nodeInfo = new HashMap<String, NodeInfo>(); for (String facilityId : facilityList) { try { synchronized (this) { nodeInfo.put(facilityId, new RepositoryControllerBean().getNode(facilityId)); } } catch (FacilityNotFound e) { // ??? } } m_log.debug("monitor start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId); String facilityId = null; /** * ? */ // ID??????? Iterator<String> itr = facilityList.iterator(); while (itr.hasNext()) { facilityId = itr.next(); if (facilityId != null && !"".equals(facilityId)) { // ???RunMonitor???? // ????????????? RunMonitorHttpScenario runMonitor = new RunMonitorHttpScenario(); // ????? runMonitor.m_monitorTypeId = this.m_monitorTypeId; runMonitor.m_monitorId = this.m_monitorId; runMonitor.m_now = this.m_now; runMonitor.m_priorityMap = this.m_priorityMap; runMonitor.setMonitorInfo(runMonitor.m_monitorTypeId, runMonitor.m_monitorId); runMonitor.setJudgementInfo(); runMonitor.setCheckInfo(); runMonitor.nodeInfo = this.nodeInfo; ecs.submit(new CallableTaskHttpScenario(runMonitor, facilityId)); taskCount++; } else { itr.remove(); } } } else { // ?? // ?? // ID????? // /?true?????ID?? facilityList = new RepositoryControllerBean().getExecTargetFacilityIdList(m_facilityId, m_monitor.getOwnerRoleId()); if (facilityList.size() != 1 || !facilityList.get(0).equals(m_facilityId)) { return true; } m_isNode = new RepositoryControllerBean().isNode(m_facilityId); // ??????? nodeInfo = new HashMap<String, NodeInfo>(); for (String facilityId : facilityList) { try { synchronized (this) { nodeInfo.put(facilityId, new RepositoryControllerBean().getNode(m_facilityId)); } } catch (FacilityNotFound e) { // ??? } } m_log.debug("monitor start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId); /** * ? */ // ???RunMonitor???? // ????????????? RunMonitorHttpScenario runMonitor = new RunMonitorHttpScenario(); // ????? runMonitor.m_isMonitorJob = this.m_isMonitorJob; runMonitor.m_monitorTypeId = this.m_monitorTypeId; runMonitor.m_monitorId = this.m_monitorId; runMonitor.m_now = this.m_now; runMonitor.m_priorityMap = this.m_priorityMap; runMonitor.setMonitorInfo(runMonitor.m_monitorTypeId, runMonitor.m_monitorId); runMonitor.setJudgementInfo(); runMonitor.setCheckInfo(); runMonitor.nodeInfo = this.nodeInfo; ecs.submit(new CallableTaskHttpScenario(runMonitor, m_facilityId)); taskCount++; } /** * ?? */ ArrayList<MonitorRunResultInfo> resultList = null; m_log.debug("total start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId); // ??? List<Sample> sampleList = new ArrayList<Sample>(); Sample sample = null; if (m_monitor.getCollectorFlg()) { sample = new Sample(HinemosTime.getDateInstance(), m_monitor.getMonitorId()); } for (int i = 0; i < taskCount; i++) { Future<ArrayList<MonitorRunResultInfo>> future = ecs.take(); resultList = future.get(); // ?? for (MonitorRunResultInfo result : resultList) { m_nodeDate = result.getNodeDate(); String facilityId = result.getFacilityId(); // ? if (!m_isMonitorJob) { if (result.getMonitorFlg()) { notify(true, facilityId, result.getCheckResult(), new Date(m_nodeDate), result); } } else { m_monitorRunResultInfo = new MonitorRunResultInfo(); m_monitorRunResultInfo.setPriority(result.getPriority()); m_monitorRunResultInfo.setCheckResult(result.getCheckResult()); m_monitorRunResultInfo.setNodeDate(m_nodeDate); m_monitorRunResultInfo .setMessageOrg(makeJobOrgMessage(result.getMessageOrg(), result.getMessage())); } // ??? if (sample != null && result.getCollectorFlg()) { int errorCode = -1; if (result.isCollectorResult()) { errorCode = CollectedDataErrorTypeConstant.NOT_ERROR; } else { errorCode = CollectedDataErrorTypeConstant.UNKNOWN; } sample.set(facilityId, m_monitor.getItemName(), result.getValue(), errorCode, result.getDisplayName()); } } } // ????? if (sample != null) { sampleList.add(sample); } if (!sampleList.isEmpty()) { CollectDataUtil.put(sampleList); } m_log.debug("monitor end : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId); return true; } catch (EntityExistsException e) { throw e; } catch (FacilityNotFound e) { throw e; } catch (InvalidRole e) { throw e; } catch (InterruptedException e) { m_log.info("runMonitorInfo() monitorTypeId = " + m_monitorTypeId + ", monitorId = " + m_monitorId + " : " + e.getClass().getSimpleName() + ", " + e.getMessage()); return false; } catch (ExecutionException e) { m_log.info("runMonitorInfo() monitorTypeId = " + m_monitorTypeId + ", monitorId = " + m_monitorId + " : " + e.getClass().getSimpleName() + ", " + e.getMessage()); return false; } }