Example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

List of usage examples for java.util.concurrent ExecutorCompletionService ExecutorCompletionService

Introduction

In this page you can find the example usage for java.util.concurrent ExecutorCompletionService ExecutorCompletionService.

Prototype

public ExecutorCompletionService(Executor executor) 

Source Link

Document

Creates an ExecutorCompletionService using the supplied executor for base task execution and a LinkedBlockingQueue as a completion queue.

Usage

From source file:com.splout.db.hadoop.GeneratorCMD.java

public int run(String[] args) throws Exception {
    JCommander jComm = new JCommander(this);
    jComm.setProgramName(//from w w  w .ja  va  2 s . co  m
            "Splout Tablespaces Generator. Generates tablespaces, ready to be deployed to a Splout Cluster.");
    try {
        jComm.parse(args);
    } catch (Throwable t) {
        t.printStackTrace();
        jComm.usage();
        return -1;
    }

    if (parallelism < 1) {
        System.err.println("Parallelism must be greater than 0.");
        System.exit(1);
    }

    log.info("Parsing input parameters...");

    // All the tablespaces that will be generated and deployed atomically, hashed by their name
    // We generate this first so we can detect errors in the configuration before even using Hadoop
    Map<String, TablespaceSpec> tablespacesToGenerate = new HashMap<String, TablespaceSpec>();

    // Partition maps to reuse at indexation. Used when sampling is skipped.
    final Map<String, PartitionMap> partitionMapsToReuse = new HashMap<String, PartitionMap>();

    for (String tablespaceFile : tablespaceFiles) {
        Path file = new Path(tablespaceFile);
        FileSystem fS = FileSystem.get(file.toUri(), getConf());

        if (!fS.exists(file)) {
            throw new IllegalArgumentException("Config input file: " + file + " doesn't exist!");
        }

        String strContents = HadoopUtils.fileToString(fS, file);
        JSONTablespaceDefinition def = JSONSerDe.deSer(strContents, JSONTablespaceDefinition.class);
        TablespaceSpec spec = def.build(conf);
        String name = def.getName();

        tablespacesToGenerate.put(name, spec);

        // Reusing partition maps?
        if (qnodeURL != null) {
            partitionMapsToReuse.put(name, retrievePartitionMapfromQNode(name));
        }
    }

    if (!FileSystem.getLocal(conf).equals(FileSystem.get(conf))) {
        File nativeLibs = new File("native");
        if (nativeLibs.exists()) {
            SploutHadoopConfiguration.addSQLite4JavaNativeLibsToDC(conf);
        }
    }

    Path out = new Path(output);
    FileSystem outFs = out.getFileSystem(getConf());
    HadoopUtils.deleteIfExists(outFs, out);

    ExecutorService executor = Executors.newFixedThreadPool(parallelism);
    ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(executor);
    ArrayList<Future<Boolean>> generatorFutures = new ArrayList<Future<Boolean>>();

    // Generate each tablespace
    for (final Map.Entry<String, TablespaceSpec> tablespace : tablespacesToGenerate.entrySet()) {
        Path tablespaceOut = new Path(out, tablespace.getKey());
        TablespaceSpec spec = tablespace.getValue();

        log.info("Generating view with Hadoop (" + tablespace.getKey() + ")");
        final TablespaceGenerator viewGenerator = new TablespaceGenerator(spec, tablespaceOut, this.getClass());

        generatorFutures.add(ecs.submit(new Callable<Boolean>() {
            @Override
            public Boolean call() throws Exception {
                if (qnodeURL == null) {
                    viewGenerator.generateView(conf, samplingType, new TupleSampler.RandomSamplingOptions());
                    return true;
                } else {
                    viewGenerator.generateView(conf, partitionMapsToReuse.get(tablespace.getKey()));
                    return true;
                }
            }
        }));
    }

    // Waiting all tasks to finish.
    for (int i = 0; i < tablespacesToGenerate.size(); i++) {
        // Get will throw an exception if the callable returned it.
        try {
            ecs.take().get();
        } catch (ExecutionException e) {
            // One job was wrong. Stopping the rest.
            for (Future<Boolean> task : generatorFutures) {
                task.cancel(true);
            }
            executor.shutdown();
            throw e;
        }
    }

    executor.shutdown();

    log.info("Done!");
    return 0;
}

From source file:org.apache.flume.channel.kafka.TestKafkaChannel.java

/**
 * This method starts a channel, puts events into it. The channel is then
 * stopped and restarted. Then we check to make sure if all events we put
 * come out. Optionally, 10 events are rolled back,
 * and optionally we restart the agent immediately after and we try to pull it
 * out.//from w ww.j  a  va 2  s  .  c o m
 *
 * @param rollback
 * @param retryAfterRollback
 * @throws Exception
 */
private void doTestStopAndStart(boolean rollback, boolean retryAfterRollback) throws Exception {
    final KafkaChannel channel = startChannel(true);
    ExecutorService underlying = Executors.newCachedThreadPool();
    ExecutorCompletionService<Void> submitterSvc = new ExecutorCompletionService<Void>(underlying);
    final List<List<Event>> events = createBaseList();
    putEvents(channel, events, submitterSvc);
    int completed = 0;
    wait(submitterSvc, 5);
    channel.stop();
    final KafkaChannel channel2 = startChannel(true);
    int total = 50;
    if (rollback && !retryAfterRollback) {
        total = 40;
    }
    final List<Event> eventsPulled = pullEvents(channel2, submitterSvc, total, rollback, retryAfterRollback);
    wait(submitterSvc, 5);
    channel2.stop();
    if (!retryAfterRollback && rollback) {
        final KafkaChannel channel3 = startChannel(true);
        int expectedRemaining = 50 - eventsPulled.size();
        final List<Event> eventsPulled2 = pullEvents(channel3, submitterSvc, expectedRemaining, false, false);
        wait(submitterSvc, 5);
        Assert.assertEquals(expectedRemaining, eventsPulled2.size());
        eventsPulled.addAll(eventsPulled2);
        channel3.stop();
    }
    underlying.shutdownNow();
    verify(eventsPulled);
}

From source file:com.opengamma.integration.viewer.status.impl.ViewStatusCalculationWorker.java

public ViewStatusResultAggregator run() {
    ViewStatusResultAggregator aggregator = new ViewStatusResultAggregatorImpl();
    CompletionService<PerViewStatusResult> completionService = new ExecutorCompletionService<PerViewStatusResult>(
            _executor);//ww w  .  jav a 2s.  c om
    //submit task to executor to run partitioned by security type
    for (String securityType : _valueRequirementBySecType.keySet()) {
        Collection<String> valueRequirements = _valueRequirementBySecType.get(securityType);
        completionService.submit(new ViewStatusCalculationTask(_toolContext, _portfolioId, _user, securityType,
                valueRequirements, _marketDataSpecification));
    }
    try {
        // process all completed task
        for (int i = 0; i < _valueRequirementBySecType.size(); i++) {
            Future<PerViewStatusResult> futureTask = completionService.take();
            PerViewStatusResult perViewStatusResult = futureTask.get();
            for (ViewStatusKey viewStatusKey : perViewStatusResult.keySet()) {
                aggregator.putStatus(viewStatusKey, perViewStatusResult.get(viewStatusKey));
            }

        }
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    } catch (ExecutionException ex) {
        throw new OpenGammaRuntimeException("Error running View status report", ex.getCause());
    }
    return aggregator;
}

From source file:pl.edu.icm.cermine.libsvm.parameters.SVMParameterFinder.java

public void run(String inputFile, String ext, int threads, int kernel, int degree, int minc, int maxc, int ming,
        int maxg) throws AnalysisException, InterruptedException, ExecutionException {
    List<TrainingSample<BxZoneLabel>> samples = getSamples(inputFile, ext);

    ExecutorService executor = Executors.newFixedThreadPool(3);
    CompletionService<EvaluationParams> completionService = new ExecutorCompletionService<EvaluationParams>(
            executor);/*from w  w w .jav a  2  s . com*/

    double bestRate = 0;
    int bestclog = 0;
    int bestglog = 0;

    int submitted = 0;

    for (int clog = minc; clog <= maxc; clog++) {
        for (int glog = maxg; glog >= ming; glog--) {
            completionService.submit(new Evaluator(samples, new EvaluationParams(clog, glog), kernel, degree));
            submitted++;
        }
    }

    while (submitted > 0) {
        Future<EvaluationParams> f1 = completionService.take();
        EvaluationParams p = f1.get();
        if (p.rate > bestRate) {
            bestRate = p.rate;
            bestclog = p.clog;
            bestglog = p.glog;
        }
        System.out.println("Gamma: " + p.glog + ", C: " + p.clog + ", rate: " + p.rate + " (Best: " + bestglog
                + " " + bestclog + " " + bestRate + ")");
        submitted--;
    }

    executor.shutdown();
}

From source file:org.commonjava.maven.galley.internal.TransferManagerImpl.java

@PostConstruct
public void init() {
    batchExecutor = new ExecutorCompletionService<>(executorService);
}

From source file:org.nickelproject.nickel.blobStore.S3BlobStore.java

private void putMultiPartByteArray(final BlobRef blobRef, final byte[] pBytes) {
    final CompletionService<PartETag> completionService = new ExecutorCompletionService<PartETag>(executor);
    final String key = blobRef.toString();
    final PartTransferInterface partGetter = RetryProxy.newInstance(PartTransferInterface.class,
            new PartTransfer());
    final String uploadId = partGetter.initiateUpload(key);
    int partNumber = 1;
    for (int start = 0; start < pBytes.length; start += uploadPartSize) {
        final long length = Math.min(uploadPartSize, pBytes.length - start);
        completionService.submit(new PutPartCallable(start, (int) length, pBytes, partNumber++, key, uploadId,
                start + uploadPartSize >= pBytes.length));
    }/*  w w  w  .ja v  a 2  s  . c  o  m*/
    final List<PartETag> partETags = Lists.newArrayList();
    for (int i = 1; i < partNumber; i++) {
        try {
            partETags.add(completionService.take().get());
        } catch (final InterruptedException e1) {
            Thread.currentThread().interrupt();
            throw RethrownException.rethrow(e1);
        } catch (final ExecutionException e2) {
            throw RethrownException.rethrow(e2);
        }
    }
    partGetter.completeUpload(key, uploadId, partETags);
}

From source file:com.netflix.curator.framework.recipes.leader.TestLeaderLatch.java

@Test
public void testWaiting() throws Exception {
    final int PARTICIPANT_QTY = 10;

    ExecutorService executorService = Executors.newFixedThreadPool(PARTICIPANT_QTY);
    ExecutorCompletionService<Void> service = new ExecutorCompletionService<Void>(executorService);

    final Timing timing = new Timing();
    final CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(),
            timing.session(), timing.connection(), new RetryOneTime(1));
    try {/*from  ww w  .  j  a  va 2 s.co m*/
        client.start();

        final AtomicBoolean thereIsALeader = new AtomicBoolean(false);
        for (int i = 0; i < PARTICIPANT_QTY; ++i) {
            service.submit(new Callable<Void>() {
                @Override
                public Void call() throws Exception {
                    LeaderLatch latch = new LeaderLatch(client, PATH_NAME);
                    try {
                        latch.start();
                        Assert.assertTrue(latch.await(timing.forWaiting().seconds(), TimeUnit.SECONDS));
                        Assert.assertTrue(thereIsALeader.compareAndSet(false, true));
                        Thread.sleep((int) (10 * Math.random()));
                    } finally {
                        thereIsALeader.set(false);
                        latch.close();
                    }
                    return null;
                }
            });
        }

        for (int i = 0; i < PARTICIPANT_QTY; ++i) {
            service.take().get();
        }
    } finally {
        executorService.shutdown();
        IOUtils.closeQuietly(client);
    }
}

From source file:org.tinymediamanager.scraper.imdb.ImdbMetadataProvider.java

@Override
public MediaMetadata getMetadata(MediaScrapeOptions options) throws Exception {
    LOGGER.debug("getMetadata() " + options.toString());
    // check if there is a md in the result
    if (options.getResult() != null && options.getResult().getMetadata() != null) {
        LOGGER.debug("IMDB: getMetadata from cache: " + options.getResult());
        return options.getResult().getMetadata();
    }/*from   w w  w.  j a v  a  2 s.c  o m*/

    MediaMetadata md = new MediaMetadata(providerInfo.getId());
    String imdbId = "";

    // imdbId from searchResult
    if (options.getResult() != null) {
        imdbId = options.getResult().getIMDBId();
    }

    // imdbid from scraper option
    if (!MetadataUtil.isValidImdbId(imdbId)) {
        imdbId = options.getImdbId();
    }

    if (!MetadataUtil.isValidImdbId(imdbId)) {
        return md;
    }

    LOGGER.debug("IMDB: getMetadata(imdbId): " + imdbId);
    md.setId(MediaMetadata.IMDBID, imdbId);

    ExecutorCompletionService<Document> compSvcImdb = new ExecutorCompletionService<Document>(executor);
    ExecutorCompletionService<MediaMetadata> compSvcTmdb = new ExecutorCompletionService<MediaMetadata>(
            executor);

    // worker for imdb request (/combined) (everytime from akas.imdb.com)
    // StringBuilder sb = new StringBuilder(imdbSite.getSite());
    StringBuilder sb = new StringBuilder(ImdbSiteDefinition.IMDB_COM.getSite());
    sb.append("title/");
    sb.append(imdbId);
    sb.append("/combined");
    Callable<Document> worker = new ImdbWorker(sb.toString(), options.getLanguage().name(),
            options.getCountry().getAlpha2());
    Future<Document> futureCombined = compSvcImdb.submit(worker);

    // worker for imdb request (/plotsummary) (from chosen site)
    Future<Document> futurePlotsummary = null;
    sb = new StringBuilder(imdbSite.getSite());
    sb.append("title/");
    sb.append(imdbId);
    sb.append("/plotsummary");

    worker = new ImdbWorker(sb.toString(), options.getLanguage().name(), options.getCountry().getAlpha2());
    futurePlotsummary = compSvcImdb.submit(worker);

    // worker for tmdb request
    Future<MediaMetadata> futureTmdb = null;
    if (options.isScrapeImdbForeignLanguage() || options.isScrapeCollectionInfo()) {
        Callable<MediaMetadata> worker2 = new TmdbWorker(imdbId, options.getLanguage(), options.getCountry());
        futureTmdb = compSvcTmdb.submit(worker2);
    }

    Document doc;
    doc = futureCombined.get();

    /*
     * title and year have the following structure
     * 
     * <div id="tn15title"><h1>Merida - Legende der Highlands <span>(<a href="/year/2012/">2012</a>) <span class="pro-link">...</span> <span
     * class="title-extra">Brave <i>(original title)</i></span> </span></h1> </div>
     */

    // parse title and year
    Element title = doc.getElementById("tn15title");
    if (title != null) {
        Element element = null;
        // title
        Elements elements = title.getElementsByTag("h1");
        if (elements.size() > 0) {
            element = elements.first();
            String movieTitle = cleanString(element.ownText());
            md.storeMetadata(MediaMetadata.TITLE, movieTitle);
        }

        // year
        elements = title.getElementsByTag("span");
        if (elements.size() > 0) {
            element = elements.first();
            String content = element.text();

            // search year
            Pattern yearPattern = Pattern.compile("\\(([0-9]{4})|/\\)");
            Matcher matcher = yearPattern.matcher(content);
            while (matcher.find()) {
                if (matcher.group(1) != null) {
                    String movieYear = matcher.group(1);
                    md.storeMetadata(MediaMetadata.YEAR, movieYear);
                    break;
                }
            }
        }

        // original title
        elements = title.getElementsByAttributeValue("class", "title-extra");
        if (elements.size() > 0) {
            element = elements.first();
            String content = element.text();
            content = content.replaceAll("\\(original title\\)", "").trim();
            md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, content);
        }
    }

    // poster
    Element poster = doc.getElementById("primary-poster");
    if (poster != null) {
        String posterUrl = poster.attr("src");
        posterUrl = posterUrl.replaceAll("SX[0-9]{2,4}_", "SX400_");
        posterUrl = posterUrl.replaceAll("SY[0-9]{2,4}_", "SY400_");
        processMediaArt(md, MediaArtworkType.POSTER, "Poster", posterUrl);
    }

    /*
     * <div class="starbar-meta"> <b>7.4/10</b> &nbsp;&nbsp;<a href="ratings" class="tn15more">52,871 votes</a>&nbsp;&raquo; </div>
     */

    // rating and rating count
    Element ratingElement = doc.getElementById("tn15rating");
    if (ratingElement != null) {
        Elements elements = ratingElement.getElementsByClass("starbar-meta");
        if (elements.size() > 0) {
            Element div = elements.get(0);

            // rating comes in <b> tag
            Elements b = div.getElementsByTag("b");
            if (b.size() == 1) {
                String ratingAsString = b.text();
                Pattern ratingPattern = Pattern.compile("([0-9]\\.[0-9])/10");
                Matcher matcher = ratingPattern.matcher(ratingAsString);
                while (matcher.find()) {
                    if (matcher.group(1) != null) {
                        float rating = 0;
                        try {
                            rating = Float.valueOf(matcher.group(1));
                        } catch (Exception e) {
                        }
                        md.storeMetadata(MediaMetadata.RATING, rating);
                        break;
                    }
                }
            }

            // count
            Elements a = div.getElementsByAttributeValue("href", "ratings");
            if (a.size() == 1) {
                String countAsString = a.text().replaceAll("[.,]|votes", "").trim();
                int voteCount = 0;
                try {
                    voteCount = Integer.parseInt(countAsString);
                } catch (Exception e) {
                }
                md.storeMetadata(MediaMetadata.VOTE_COUNT, voteCount);
            }
        }

        // top250
        elements = ratingElement.getElementsByClass("starbar-special");
        if (elements.size() > 0) {
            Elements a = elements.get(0).getElementsByTag("a");
            if (a.size() > 0) {
                Element anchor = a.get(0);
                Pattern topPattern = Pattern.compile("Top 250: #([0-9]{1,3})");
                Matcher matcher = topPattern.matcher(anchor.ownText());
                while (matcher.find()) {
                    if (matcher.group(1) != null) {
                        int top250 = 0;
                        try {
                            top250 = Integer.parseInt(matcher.group(1));
                        } catch (Exception e) {
                        }
                        md.storeMetadata(MediaMetadata.TOP_250, top250);
                    }
                }
            }
        }
    }

    // parse all items coming by <div class="info">
    Elements elements = doc.getElementsByClass("info");
    for (Element element : elements) {
        // only parse divs
        if (!"div".equals(element.tag().getName())) {
            continue;
        }

        // elements with h5 are the titles of the values
        Elements h5 = element.getElementsByTag("h5");
        if (h5.size() > 0) {
            Element firstH5 = h5.first();
            String h5Title = firstH5.text();

            // release date
            /*
             * <div class="info"><h5>Release Date:</h5><div class="info-content">5 January 1996 (USA)<a class="tn15more inline"
             * href="/title/tt0114746/releaseinfo"
             * onclick="(new Image()).src='/rg/title-tease/releasedates/images/b.gif?link=/title/tt0114746/releaseinfo';"> See more</a>&nbsp;</div></div>
             */
            if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getReleaseDate() + ".*")) {
                Elements div = element.getElementsByClass("info-content");
                if (div.size() > 0) {
                    Element releaseDateElement = div.first();
                    String releaseDate = cleanString(releaseDateElement.ownText().replaceAll("", ""));
                    Pattern pattern = Pattern.compile("(.*)\\(.*\\)");
                    Matcher matcher = pattern.matcher(releaseDate);
                    if (matcher.find()) {
                        try {
                            SimpleDateFormat sdf = new SimpleDateFormat("d MMM yyyy");
                            Date parsedDate = sdf.parse(matcher.group(1));
                            sdf = new SimpleDateFormat("dd-MM-yyyy");
                            md.storeMetadata(MediaMetadata.RELEASE_DATE, sdf.format(parsedDate));
                        } catch (Exception e) {
                        }
                    }
                }
            }

            /*
             * <div class="info"><h5>Tagline:</h5><div class="info-content"> (7) To Defend Us... <a class="tn15more inline"
             * href="/title/tt0472033/taglines" onClick= "(new Image()).src='/rg/title-tease/taglines/images/b.gif?link=/title/tt0472033/taglines';" >See
             * more</a>&nbsp;&raquo; </div></div>
             */
            // tagline
            if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getTagline() + ".*")
                    && !options.isScrapeImdbForeignLanguage()) {
                Elements div = element.getElementsByClass("info-content");
                if (div.size() > 0) {
                    Element taglineElement = div.first();
                    String tagline = cleanString(taglineElement.ownText().replaceAll("", ""));
                    md.storeMetadata(MediaMetadata.TAGLINE, tagline);
                }
            }

            /*
             * <div class="info-content"><a href="/Sections/Genres/Animation/">Animation</a> | <a href="/Sections/Genres/Action/">Action</a> | <a
             * href="/Sections/Genres/Adventure/">Adventure</a> | <a href="/Sections/Genres/Fantasy/">Fantasy</a> | <a
             * href="/Sections/Genres/Mystery/">Mystery</a> | <a href="/Sections/Genres/Sci-Fi/">Sci-Fi</a> | <a
             * href="/Sections/Genres/Thriller/">Thriller</a> <a class="tn15more inline" href="/title/tt0472033/keywords" onClick=
             * "(new Image()).src='/rg/title-tease/keywords/images/b.gif?link=/title/tt0472033/keywords';" > See more</a>&nbsp;&raquo; </div>
             */
            // genres are only scraped from akas.imdb.com
            if (h5Title.matches("(?i)" + imdbSite.getGenre() + "(.*)")) {
                Elements div = element.getElementsByClass("info-content");
                if (div.size() > 0) {
                    Elements a = div.first().getElementsByTag("a");
                    for (Element anchor : a) {
                        if (anchor.attr("href").matches("/Sections/Genres/.*")) {
                            md.addGenre(getTmmGenre(anchor.ownText()));
                        }
                    }
                }
            }
            // }

            /*
             * <div class="info"><h5>Runtime:</h5><div class="info-content">162 min | 171 min (special edition) | 178 min (extended cut)</div></div>
             */
            // runtime
            // if (h5Title.matches("(?i)" + imdbSite.getRuntime() + ".*")) {
            if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getRuntime() + ".*")) {
                Elements div = element.getElementsByClass("info-content");
                if (div.size() > 0) {
                    Element taglineElement = div.first();
                    String first = taglineElement.ownText().split("\\|")[0];
                    String runtimeAsString = cleanString(first.replaceAll("min", ""));
                    int runtime = 0;
                    try {
                        runtime = Integer.parseInt(runtimeAsString);
                    } catch (Exception e) {
                        // try to filter out the first number we find
                        Pattern runtimePattern = Pattern.compile("([0-9]{2,3})");
                        Matcher matcher = runtimePattern.matcher(runtimeAsString);
                        if (matcher.find()) {
                            runtime = Integer.parseInt(matcher.group(0));
                        }
                    }
                    md.storeMetadata(MediaMetadata.RUNTIME, runtime);
                }
            }

            /*
             * <div class="info"><h5>Country:</h5><div class="info-content"><a href="/country/fr">France</a> | <a href="/country/es">Spain</a> | <a
             * href="/country/it">Italy</a> | <a href="/country/hu">Hungary</a></div></div>
             */
            // country
            if (h5Title.matches("(?i)Country.*")) {
                Elements a = element.getElementsByTag("a");
                String countries = "";
                for (Element anchor : a) {
                    Pattern pattern = Pattern.compile("/country/(.*)");
                    Matcher matcher = pattern.matcher(anchor.attr("href"));
                    if (matcher.matches()) {
                        String country = matcher.group(1);
                        if (StringUtils.isNotEmpty(countries)) {
                            countries += ", ";
                        }
                        countries += country.toUpperCase();
                    }
                }
                md.storeMetadata(MediaMetadata.COUNTRY, countries);
            }

            /*
             * <div class="info"><h5>Language:</h5><div class="info-content"><a href="/language/en">English</a> | <a href="/language/de">German</a> | <a
             * href="/language/fr">French</a> | <a href="/language/it">Italian</a></div>
             */
            // Spoken languages
            if (h5Title.matches("(?i)Language.*")) {
                Elements a = element.getElementsByTag("a");
                String spokenLanguages = "";
                for (Element anchor : a) {
                    Pattern pattern = Pattern.compile("/language/(.*)");
                    Matcher matcher = pattern.matcher(anchor.attr("href"));
                    if (matcher.matches()) {
                        String langu = matcher.group(1);
                        if (StringUtils.isNotEmpty(spokenLanguages)) {
                            spokenLanguages += ", ";
                        }
                        spokenLanguages += langu;
                    }
                }
                md.storeMetadata(MediaMetadata.SPOKEN_LANGUAGES, spokenLanguages);
            }

            /*
             * <div class="info"><h5>Certification:</h5><div class="info-content"><a href="/search/title?certificates=us:pg">USA:PG</a> <i>(certificate
             * #47489)</i> | <a href="/search/title?certificates=ca:pg">Canada:PG</a> <i>(Ontario)</i> | <a
             * href="/search/title?certificates=au:pg">Australia:PG</a> | <a href="/search/title?certificates=in:u">India:U</a> | <a
             * href="/search/title?certificates=ie:pg">Ireland:PG</a> ...</div></div>
             */
            // certification
            // if (h5Title.matches("(?i)" + imdbSite.getCertification() + ".*")) {
            if (h5Title.matches("(?i)" + ImdbSiteDefinition.IMDB_COM.getCertification() + ".*")) {
                Elements a = element.getElementsByTag("a");
                for (Element anchor : a) {
                    // certification for the right country
                    if (anchor.attr("href").matches(
                            "(?i)/search/title\\?certificates=" + options.getCountry().getAlpha2() + ".*")) {
                        Pattern certificationPattern = Pattern.compile(".*:(.*)");
                        Matcher matcher = certificationPattern.matcher(anchor.ownText());
                        Certification certification = null;
                        while (matcher.find()) {
                            if (matcher.group(1) != null) {
                                certification = Certification.getCertification(options.getCountry(),
                                        matcher.group(1));
                            }
                        }

                        if (certification != null) {
                            md.addCertification(certification);
                            break;
                        }
                    }
                }
            }
        }

        /*
         * <div id="director-info" class="info"> <h5>Director:</h5> <div class="info-content"><a href="/name/nm0000416/" onclick=
         * "(new Image()).src='/rg/directorlist/position-1/images/b.gif?link=name/nm0000416/';" >Terry Gilliam</a><br/> </div> </div>
         */
        // director
        if ("director-info".equals(element.id())) {
            Elements a = element.getElementsByTag("a");
            for (Element anchor : a) {
                if (anchor.attr("href").matches("/name/nm.*")) {
                    MediaCastMember cm = new MediaCastMember(CastType.DIRECTOR);
                    cm.setName(anchor.ownText());
                    md.addCastMember(cm);
                }
            }
        }
    }

    /*
     * <table class="cast"> <tr class="odd"><td class="hs"><a href="http://pro.imdb.com/widget/resume_redirect/" onClick=
     * "(new Image()).src='/rg/resume/prosystem/images/b.gif?link=http://pro.imdb.com/widget/resume_redirect/';" ><img src=
     * "http://i.media-imdb.com/images/SF9113d6f5b7cb1533c35313ccd181a6b1/tn15/no_photo.png" width="25" height="31" border="0"></td><td class="nm"><a
     * href="/name/nm0577828/" onclick= "(new Image()).src='/rg/castlist/position-1/images/b.gif?link=/name/nm0577828/';" >Joseph Melito</a></td><td
     * class="ddd"> ... </td><td class="char"><a href="/character/ch0003139/">Young Cole</a></td></tr> <tr class="even"><td class="hs"><a
     * href="/name/nm0000246/" onClick= "(new Image()).src='/rg/title-tease/tinyhead/images/b.gif?link=/name/nm0000246/';" ><img src=
     * "http://ia.media-imdb.com/images/M/MV5BMjA0MjMzMTE5OF5BMl5BanBnXkFtZTcwMzQ2ODE3Mw@@._V1._SY30_SX23_.jpg" width="23" height="32"
     * border="0"></a><br></td><td class="nm"><a href="/name/nm0000246/" onclick=
     * "(new Image()).src='/rg/castlist/position-2/images/b.gif?link=/name/nm0000246/';" >Bruce Willis</a></td><td class="ddd"> ... </td><td
     * class="char"><a href="/character/ch0003139/">James Cole</a></td></tr> <tr class="odd"><td class="hs"><a href="/name/nm0781218/" onClick=
     * "(new Image()).src='/rg/title-tease/tinyhead/images/b.gif?link=/name/nm0781218/';" ><img src=
     * "http://ia.media-imdb.com/images/M/MV5BODI1MTA2MjkxM15BMl5BanBnXkFtZTcwMTcwMDg2Nw@@._V1._SY30_SX23_.jpg" width="23" height="32"
     * border="0"></a><br></td><td class="nm"><a href="/name/nm0781218/" onclick=
     * "(new Image()).src='/rg/castlist/position-3/images/b.gif?link=/name/nm0781218/';" >Jon Seda</a></td><td class="ddd"> ... </td><td
     * class="char"><a href="/character/ch0003143/">Jose</a></td></tr>...</table>
     */
    // cast
    elements = doc.getElementsByClass("cast");
    if (elements.size() > 0) {
        Elements tr = elements.get(0).getElementsByTag("tr");
        for (Element row : tr) {
            Elements td = row.getElementsByTag("td");
            MediaCastMember cm = new MediaCastMember();
            for (Element column : td) {
                // actor thumb
                if (column.hasClass("hs")) {
                    Elements img = column.getElementsByTag("img");
                    if (img.size() > 0) {
                        String thumbUrl = img.get(0).attr("src");
                        if (thumbUrl.contains("no_photo.png")) {
                            cm.setImageUrl("");
                        } else {
                            thumbUrl = thumbUrl.replaceAll("SX[0-9]{2,4}_", "SX400_");
                            thumbUrl = thumbUrl.replaceAll("SY[0-9]{2,4}_", "");
                            cm.setImageUrl(thumbUrl);
                        }
                    }
                }
                // actor name
                if (column.hasClass("nm")) {
                    cm.setName(cleanString(column.text()));
                }
                // character
                if (column.hasClass("char")) {
                    cm.setCharacter(cleanString(column.text()));
                }
            }
            if (StringUtils.isNotEmpty(cm.getName()) && StringUtils.isNotEmpty(cm.getCharacter())) {
                cm.setType(CastType.ACTOR);
                md.addCastMember(cm);
            }
        }
    }

    Element content = doc.getElementById("tn15content");
    if (content != null) {
        elements = content.getElementsByTag("table");
        for (Element table : elements) {
            // writers
            if (table.text().contains(ImdbSiteDefinition.IMDB_COM.getWriter())) {
                Elements anchors = table.getElementsByTag("a");
                for (Element anchor : anchors) {
                    if (anchor.attr("href").matches("/name/nm.*")) {
                        MediaCastMember cm = new MediaCastMember(CastType.WRITER);
                        cm.setName(anchor.ownText());
                        md.addCastMember(cm);
                    }
                }
            }

            // producers
            if (table.text().contains(ImdbSiteDefinition.IMDB_COM.getProducers())) {
                Elements rows = table.getElementsByTag("tr");
                for (Element row : rows) {
                    if (row.text().contains(ImdbSiteDefinition.IMDB_COM.getProducers())) {
                        continue;
                    }
                    Elements columns = row.children();
                    if (columns.size() == 0) {
                        continue;
                    }
                    MediaCastMember cm = new MediaCastMember(CastType.PRODUCER);
                    String name = cleanString(columns.get(0).text());
                    if (StringUtils.isBlank(name)) {
                        continue;
                    }
                    cm.setName(name);
                    if (columns.size() >= 3) {
                        cm.setPart(cleanString(columns.get(2).text()));
                    }
                    md.addCastMember(cm);
                }
            }
        }
    }

    // Production companies
    elements = doc.getElementsByClass("blackcatheader");
    for (Element blackcatheader : elements) {
        if (blackcatheader.ownText().equals(ImdbSiteDefinition.IMDB_COM.getProductionCompanies())) {
            Elements a = blackcatheader.nextElementSibling().getElementsByTag("a");
            StringBuilder productionCompanies = new StringBuilder();
            for (Element anchor : a) {
                if (StringUtils.isNotEmpty(productionCompanies)) {
                    productionCompanies.append(", ");
                }
                productionCompanies.append(anchor.ownText());
            }
            md.storeMetadata(MediaMetadata.PRODUCTION_COMPANY, productionCompanies.toString());
            break;
        }
    }

    /*
     * plot from /plotsummary
     */
    // build the url
    doc = null;
    doc = futurePlotsummary.get();

    // imdb.com has another site structure
    if (imdbSite == ImdbSiteDefinition.IMDB_COM) {
        Elements zebraList = doc.getElementsByClass("zebraList");
        if (zebraList != null && !zebraList.isEmpty()) {
            Elements odd = zebraList.get(0).getElementsByClass("odd");
            if (odd.isEmpty()) {
                odd = zebraList.get(0).getElementsByClass("even"); // sometimes imdb has even
            }
            if (odd.size() > 0) {
                Elements p = odd.get(0).getElementsByTag("p");
                if (p.size() > 0) {
                    String plot = cleanString(p.get(0).ownText());
                    md.storeMetadata(MediaMetadata.PLOT, plot);
                }
            }
        }
    } else {
        Element wiki = doc.getElementById("swiki.2.1");
        if (wiki != null) {
            String plot = cleanString(wiki.ownText());
            md.storeMetadata(MediaMetadata.PLOT, plot);
        }
    }

    // title also from chosen site if we are not scraping akas.imdb.com
    if (imdbSite != ImdbSiteDefinition.IMDB_COM) {
        title = doc.getElementById("tn15title");
        if (title != null) {
            Element element = null;
            // title
            elements = title.getElementsByClass("main");
            if (elements.size() > 0) {
                element = elements.first();
                String movieTitle = cleanString(element.ownText());
                md.storeMetadata(MediaMetadata.TITLE, movieTitle);
            }
        }
    }
    // }

    // get data from tmdb?
    if (options.isScrapeImdbForeignLanguage() || options.isScrapeCollectionInfo()) {
        MediaMetadata tmdbMd = futureTmdb.get();
        if (options.isScrapeImdbForeignLanguage() && tmdbMd != null
                && StringUtils.isNotBlank(tmdbMd.getStringValue(MediaMetadata.PLOT))) {
            // tmdbid
            md.setId(MediaMetadata.TMDBID, tmdbMd.getId(MediaMetadata.TMDBID));
            // title
            md.storeMetadata(MediaMetadata.TITLE, tmdbMd.getStringValue(MediaMetadata.TITLE));
            // original title
            md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, tmdbMd.getStringValue(MediaMetadata.ORIGINAL_TITLE));
            // tagline
            md.storeMetadata(MediaMetadata.TAGLINE, tmdbMd.getStringValue(MediaMetadata.TAGLINE));
            // plot
            md.storeMetadata(MediaMetadata.PLOT, tmdbMd.getStringValue(MediaMetadata.PLOT));
            // collection info
            md.storeMetadata(MediaMetadata.COLLECTION_NAME,
                    tmdbMd.getStringValue(MediaMetadata.COLLECTION_NAME));
            md.storeMetadata(MediaMetadata.TMDBID_SET, tmdbMd.getIntegerValue(MediaMetadata.TMDBID_SET));
        }
        if (options.isScrapeCollectionInfo() && tmdbMd != null) {
            md.storeMetadata(MediaMetadata.TMDBID_SET, tmdbMd.getIntegerValue(MediaMetadata.TMDBID_SET));
            md.storeMetadata(MediaMetadata.COLLECTION_NAME,
                    tmdbMd.getStringValue(MediaMetadata.COLLECTION_NAME));
        }
    }

    // if we have still no original title, take the title
    if (StringUtils.isBlank(md.getStringValue(MediaMetadata.ORIGINAL_TITLE))) {
        md.storeMetadata(MediaMetadata.ORIGINAL_TITLE, md.getStringValue(MediaMetadata.TITLE));
    }

    return md;
}

From source file:io.prestosql.plugin.accumulo.index.ColumnCardinalityCache.java

/**
 * Gets the cardinality for each {@link AccumuloColumnConstraint}.
 * Given constraints are expected to be indexed! Who knows what would happen if they weren't!
 *
 * @param schema Schema name/* w  ww.  jav a  2  s  . c  o m*/
 * @param table Table name
 * @param auths Scan authorizations
 * @param idxConstraintRangePairs Mapping of all ranges for a given constraint
 * @param earlyReturnThreshold Smallest acceptable cardinality to return early while other tasks complete
 * @param pollingDuration Duration for polling the cardinality completion service
 * @return An immutable multimap of cardinality to column constraint, sorted by cardinality from smallest to largest
 * @throws TableNotFoundException If the metrics table does not exist
 * @throws ExecutionException If another error occurs; I really don't even know anymore.
 */
public Multimap<Long, AccumuloColumnConstraint> getCardinalities(String schema, String table,
        Authorizations auths, Multimap<AccumuloColumnConstraint, Range> idxConstraintRangePairs,
        long earlyReturnThreshold, Duration pollingDuration) {
    // Submit tasks to the executor to fetch column cardinality, adding it to the Guava cache if necessary
    CompletionService<Pair<Long, AccumuloColumnConstraint>> executor = new ExecutorCompletionService<>(
            executorService);
    idxConstraintRangePairs.asMap().forEach((key, value) -> executor.submit(() -> {
        long cardinality = getColumnCardinality(schema, table, auths, key.getFamily(), key.getQualifier(),
                value);
        LOG.debug("Cardinality for column %s is %s", key.getName(), cardinality);
        return Pair.of(cardinality, key);
    }));

    // Create a multi map sorted by cardinality
    ListMultimap<Long, AccumuloColumnConstraint> cardinalityToConstraints = MultimapBuilder.treeKeys()
            .arrayListValues().build();
    try {
        boolean earlyReturn = false;
        int numTasks = idxConstraintRangePairs.asMap().entrySet().size();
        do {
            // Sleep for the polling duration to allow concurrent tasks to run for this time
            Thread.sleep(pollingDuration.toMillis());

            // Poll each task, retrieving the result if it is done
            for (int i = 0; i < numTasks; ++i) {
                Future<Pair<Long, AccumuloColumnConstraint>> futureCardinality = executor.poll();
                if (futureCardinality != null && futureCardinality.isDone()) {
                    Pair<Long, AccumuloColumnConstraint> columnCardinality = futureCardinality.get();
                    cardinalityToConstraints.put(columnCardinality.getLeft(), columnCardinality.getRight());
                }
            }

            // If the smallest cardinality is present and below the threshold, set the earlyReturn flag
            Optional<Entry<Long, AccumuloColumnConstraint>> smallestCardinality = cardinalityToConstraints
                    .entries().stream().findFirst();
            if (smallestCardinality.isPresent()) {
                if (smallestCardinality.get().getKey() <= earlyReturnThreshold) {
                    LOG.info("Cardinality %s, is below threshold. Returning early while other tasks finish",
                            smallestCardinality);
                    earlyReturn = true;
                }
            }
        } while (!earlyReturn && cardinalityToConstraints.entries().size() < numTasks);
    } catch (ExecutionException | InterruptedException e) {
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Exception when getting cardinality", e);
    }

    // Create a copy of the cardinalities
    return ImmutableMultimap.copyOf(cardinalityToConstraints);
}

From source file:com.clustercontrol.http.factory.RunMonitorHttpScenario.java

/**
 * []????/*from w  w w  . j av  a2s  .  c o m*/
 *
 * ??1??ID??????ID??????????
 * ????????runMonitorInfo???
 *
 */
@Override
protected boolean runMonitorInfo()
        throws FacilityNotFound, MonitorNotFound, EntityExistsException, InvalidRole, HinemosUnknown {
    m_log.debug("runMonitorInfo()");

    m_now = new Date(HinemosTime.currentTimeMillis());

    m_priorityMap = new HashMap<Integer, ArrayList<String>>();
    m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_INFO), new ArrayList<String>());
    m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_WARNING), new ArrayList<String>());
    m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_CRITICAL), new ArrayList<String>());
    m_priorityMap.put(Integer.valueOf(PriorityConstant.TYPE_UNKNOWN), new ArrayList<String>());

    try {
        // 
        boolean run = this.setMonitorInfo(m_monitorTypeId, m_monitorId);
        if (!run) {
            // ?
            return true;
        }

        // 
        setJudgementInfo();

        // ??
        setCheckInfo();

        ArrayList<String> facilityList = null;
        ExecutorCompletionService<ArrayList<MonitorRunResultInfo>> ecs = new ExecutorCompletionService<ArrayList<MonitorRunResultInfo>>(
                ParallelExecution.instance().getExecutorService());
        int taskCount = 0;

        if (!m_isMonitorJob) {
            // ??
            // ID?????
            // /?true?????ID??
            facilityList = new RepositoryControllerBean().getExecTargetFacilityIdList(m_facilityId,
                    m_monitor.getOwnerRoleId());
            if (facilityList.size() == 0) {
                return true;
            }

            m_isNode = new RepositoryControllerBean().isNode(m_facilityId);

            // ???????
            nodeInfo = new HashMap<String, NodeInfo>();
            for (String facilityId : facilityList) {
                try {
                    synchronized (this) {
                        nodeInfo.put(facilityId, new RepositoryControllerBean().getNode(facilityId));
                    }
                } catch (FacilityNotFound e) {
                    // ???
                }
            }

            m_log.debug("monitor start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId);

            String facilityId = null;

            /**
             * ?
             */
            // ID???????
            Iterator<String> itr = facilityList.iterator();
            while (itr.hasNext()) {
                facilityId = itr.next();
                if (facilityId != null && !"".equals(facilityId)) {

                    // ???RunMonitor????
                    // ?????????????
                    RunMonitorHttpScenario runMonitor = new RunMonitorHttpScenario();

                    // ?????
                    runMonitor.m_monitorTypeId = this.m_monitorTypeId;
                    runMonitor.m_monitorId = this.m_monitorId;
                    runMonitor.m_now = this.m_now;
                    runMonitor.m_priorityMap = this.m_priorityMap;
                    runMonitor.setMonitorInfo(runMonitor.m_monitorTypeId, runMonitor.m_monitorId);
                    runMonitor.setJudgementInfo();
                    runMonitor.setCheckInfo();
                    runMonitor.nodeInfo = this.nodeInfo;

                    ecs.submit(new CallableTaskHttpScenario(runMonitor, facilityId));
                    taskCount++;
                } else {
                    itr.remove();
                }
            }
        } else {
            // ??
            // ??
            // ID?????
            // /?true?????ID??
            facilityList = new RepositoryControllerBean().getExecTargetFacilityIdList(m_facilityId,
                    m_monitor.getOwnerRoleId());
            if (facilityList.size() != 1 || !facilityList.get(0).equals(m_facilityId)) {
                return true;
            }

            m_isNode = new RepositoryControllerBean().isNode(m_facilityId);

            // ???????
            nodeInfo = new HashMap<String, NodeInfo>();
            for (String facilityId : facilityList) {
                try {
                    synchronized (this) {
                        nodeInfo.put(facilityId, new RepositoryControllerBean().getNode(m_facilityId));
                    }
                } catch (FacilityNotFound e) {
                    // ???
                }
            }
            m_log.debug("monitor start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId);

            /**
             * ?
             */
            // ???RunMonitor????
            // ?????????????
            RunMonitorHttpScenario runMonitor = new RunMonitorHttpScenario();

            // ?????
            runMonitor.m_isMonitorJob = this.m_isMonitorJob;
            runMonitor.m_monitorTypeId = this.m_monitorTypeId;
            runMonitor.m_monitorId = this.m_monitorId;
            runMonitor.m_now = this.m_now;
            runMonitor.m_priorityMap = this.m_priorityMap;
            runMonitor.setMonitorInfo(runMonitor.m_monitorTypeId, runMonitor.m_monitorId);
            runMonitor.setJudgementInfo();
            runMonitor.setCheckInfo();
            runMonitor.nodeInfo = this.nodeInfo;

            ecs.submit(new CallableTaskHttpScenario(runMonitor, m_facilityId));
            taskCount++;
        }
        /**
         * ??
         */
        ArrayList<MonitorRunResultInfo> resultList = null;

        m_log.debug("total start : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId);

        // ???
        List<Sample> sampleList = new ArrayList<Sample>();
        Sample sample = null;
        if (m_monitor.getCollectorFlg()) {
            sample = new Sample(HinemosTime.getDateInstance(), m_monitor.getMonitorId());
        }

        for (int i = 0; i < taskCount; i++) {
            Future<ArrayList<MonitorRunResultInfo>> future = ecs.take();
            resultList = future.get(); // ??

            for (MonitorRunResultInfo result : resultList) {
                m_nodeDate = result.getNodeDate();

                String facilityId = result.getFacilityId();

                // ?
                if (!m_isMonitorJob) {
                    if (result.getMonitorFlg()) {
                        notify(true, facilityId, result.getCheckResult(), new Date(m_nodeDate), result);
                    }
                } else {
                    m_monitorRunResultInfo = new MonitorRunResultInfo();
                    m_monitorRunResultInfo.setPriority(result.getPriority());
                    m_monitorRunResultInfo.setCheckResult(result.getCheckResult());
                    m_monitorRunResultInfo.setNodeDate(m_nodeDate);
                    m_monitorRunResultInfo
                            .setMessageOrg(makeJobOrgMessage(result.getMessageOrg(), result.getMessage()));
                }

                // ???
                if (sample != null && result.getCollectorFlg()) {
                    int errorCode = -1;
                    if (result.isCollectorResult()) {
                        errorCode = CollectedDataErrorTypeConstant.NOT_ERROR;
                    } else {
                        errorCode = CollectedDataErrorTypeConstant.UNKNOWN;
                    }
                    sample.set(facilityId, m_monitor.getItemName(), result.getValue(), errorCode,
                            result.getDisplayName());
                }
            }
        }

        // ?????
        if (sample != null) {
            sampleList.add(sample);
        }
        if (!sampleList.isEmpty()) {
            CollectDataUtil.put(sampleList);
        }
        m_log.debug("monitor end : monitorTypeId : " + m_monitorTypeId + ", monitorId : " + m_monitorId);

        return true;

    } catch (EntityExistsException e) {
        throw e;
    } catch (FacilityNotFound e) {
        throw e;
    } catch (InvalidRole e) {
        throw e;
    } catch (InterruptedException e) {
        m_log.info("runMonitorInfo() monitorTypeId = " + m_monitorTypeId + ", monitorId = " + m_monitorId
                + " : " + e.getClass().getSimpleName() + ", " + e.getMessage());
        return false;
    } catch (ExecutionException e) {
        m_log.info("runMonitorInfo() monitorTypeId = " + m_monitorTypeId + ", monitorId = " + m_monitorId
                + " : " + e.getClass().getSimpleName() + ", " + e.getMessage());
        return false;
    }
}