List of usage examples for java.util LinkedList size
int size
To view the source code for java.util LinkedList size.
Click Source Link
From source file:elh.eus.absa.Features.java
/** * POS ngram extraction from a kaf document * // w w w.ja v a2s. c o m * @param int length : which 'n' use for 'n-grams' * @param KAFDocument kafDoc : postagged kaf document to extract ngrams from. * @param boolean save : safe ngrams to file or not. * @return TreeSet<String> return lemma ngrams of length length */ public int extractPosNgrams(int length, KAFDocument kafDoc, List<String> discardPos, boolean save) { //System.err.println("POS ngram extraction: _"+length+"_"); if (length == 0) { return 0; } int sentNum = kafDoc.getSentences().size(); for (int s = 0; s < sentNum; s++) { LinkedList<String> ngrams = new LinkedList<String>(); for (Term term : kafDoc.getTermsBySent(s)) { if (ngrams.size() >= length) { ngrams.removeFirst(); } if (!discardPos.contains(term.getPos())) { ngrams.add(term.getPos()); } // add ngrams to the feature list for (int i = 0; i < ngrams.size(); i++) { String ng = featureFromArray(ngrams.subList(0, i + 1), "pos"); addNgram("pos", ng); } } //empty ngram list and add remaining ngrams to the feature list while (!ngrams.isEmpty()) { String ng = featureFromArray(ngrams, "pos"); addNgram("pos", ng); ngrams.removeFirst(); } } return 1; }
From source file:org.openscience.cdk.applications.taverna.weka.classification.EvaluateClassificationResultsAsPDFActivity.java
@Override public void work() throws Exception { // Get input// w w w .jav a 2s .c o m String[] options = ((String) this.getConfiguration() .getAdditionalProperty(CDKTavernaConstants.PROPERTY_SCATTER_PLOT_OPTIONS)).split(";"); List<File> modelFiles = this.getInputAsFileList(this.INPUT_PORTS[0]); List<Instances> trainDatasets = this.getInputAsList(this.INPUT_PORTS[1], Instances.class); List<Instances> testDatasets = null; if (options[0].equals("" + TEST_TRAININGSET_PORT)) { testDatasets = this.getInputAsList(this.INPUT_PORTS[2], Instances.class); } String directory = modelFiles.get(0).getParent(); // Do work ChartTool chartTool = new ChartTool(); WekaTools tools = new WekaTools(); ArrayList<String> resultFiles = new ArrayList<String>(); DefaultCategoryDataset meanClassificationChartset = new DefaultCategoryDataset(); int fileIndex = 0; while (!modelFiles.isEmpty()) { fileIndex++; List<Object> chartObjects = new LinkedList<Object>(); LinkedList<Double> trainPercentage = new LinkedList<Double>(); LinkedList<Double> testPercentage = new LinkedList<Double>(); for (int j = 0; j < trainDatasets.size(); j++) { File modelFile = modelFiles.remove(0); Classifier classifier = (Classifier) SerializationHelper.read(modelFile.getPath()); DefaultCategoryDataset chartDataset = new DefaultCategoryDataset(); String summary = ""; Instances trainset = trainDatasets.get(j); Instances tempset = Filter.useFilter(trainset, tools.getIDRemover(trainset)); Evaluation trainsetEval = new Evaluation(tempset); trainsetEval.evaluateModel(classifier, tempset); String setname = "Training set (" + String.format("%.2f", trainsetEval.pctCorrect()) + "%)"; this.createDataset(trainset, classifier, chartDataset, trainPercentage, setname); summary += "Training set:\n\n"; summary += trainsetEval.toSummaryString(true); double ratio = 100; if (testDatasets != null) { Instances testset = testDatasets.get(j); tempset = Filter.useFilter(testset, tools.getIDRemover(testset)); Evaluation testEval = new Evaluation(trainset); testEval.evaluateModel(classifier, tempset); setname = "Test set (" + String.format("%.2f", testEval.pctCorrect()) + "%)"; this.createDataset(testset, classifier, chartDataset, testPercentage, setname); summary += "\nTest set:\n\n"; summary += testEval.toSummaryString(true); ratio = trainset.numInstances() / (double) (trainset.numInstances() + testset.numInstances()) * 100; } String header = classifier.getClass().getSimpleName() + "\n Training set ratio: " + String.format("%.2f", ratio) + "\n" + modelFile.getName(); chartObjects.add(chartTool.createBarChart(header, "Class", "Correct classified (%)", chartDataset)); chartObjects.add(summary); } DefaultCategoryDataset percentageChartSet = new DefaultCategoryDataset(); double mean = 0; for (int i = 0; i < trainPercentage.size(); i++) { percentageChartSet.addValue(trainPercentage.get(i), "Training Set", "" + (i + 1)); mean += trainPercentage.get(i); } mean /= trainPercentage.size(); meanClassificationChartset.addValue(mean, "Training Set", "" + fileIndex); mean = 0; for (int i = 0; i < testPercentage.size(); i++) { percentageChartSet.addValue(testPercentage.get(i), "Test Set", "" + (i + 1)); mean += testPercentage.get(i); } mean /= testPercentage.size(); meanClassificationChartset.addValue(mean, "Test Set", "" + fileIndex); chartObjects.add(chartTool.createLineChart("Overall Percentages", "Index", "Correct Classified (%)", percentageChartSet, false, true)); File file = FileNameGenerator.getNewFile(directory, ".pdf", "ScatterPlot"); chartTool.writeChartAsPDF(file, chartObjects); resultFiles.add(file.getPath()); } JFreeChart meanChart = chartTool.createLineChart("Overall Percentages", "Model Index", "Correct Classified (%)", meanClassificationChartset, false, true); File file = FileNameGenerator.getNewFile(directory, ".pdf", "ScatterPlot"); chartTool.writeChartAsPDF(file, Collections.singletonList((Object) meanChart)); resultFiles.add(file.getPath()); // Set output this.setOutputAsStringList(resultFiles, this.OUTPUT_PORTS[0]); }
From source file:elh.eus.absa.Features.java
/** * Lemma ngram extraction from a kaf document * /*from w w w . j a v a2 s . c o m*/ * @param int length : which 'n' use for 'n-grams' * @param KAFDocument kafDoc : postagged kaf document to extract ngrams from. * @param boolean save : safe ngrams to file or not. * @return TreeSet<String> return lemma ngrams of length length */ private int extractLemmaNgrams(int length, KAFDocument kafDoc, List<String> discardPos, boolean save) { //System.err.println("lemma ngram extraction: _"+length+"_"); if (length == 0) { return 0; } int sentNum = kafDoc.getSentences().size(); for (int s = 0; s < sentNum; s++) { LinkedList<String> ngrams = new LinkedList<String>(); for (Term term : kafDoc.getTermsBySent(s)) { if (ngrams.size() >= length) { ngrams.removeFirst(); } //if no alphanumeric char is present discard the element as invalid ngram. Or if it has a PoS tag that //should be discarded String lCurrent = term.getLemma(); if ((!discardPos.contains(term.getPos())) && (!lCurrent.matches("[^\\p{L}\\p{M}\\p{Nd}\\p{InEmoticons}]+")) && (lCurrent.length() > 1)) { ngrams.add(lCurrent); //ngrams.add(normalize(term.getLemma(), params.getProperty("normalization", "none"))); } //certain punctuation marks and emoticons are allowed as lemmas else if ((lCurrent.length() <= 2) && (lCurrent.matches("[,;.?!]"))) { ngrams.add(lCurrent); } // add ngrams to the feature list for (int i = 0; i < ngrams.size(); i++) { String ng = featureFromArray(ngrams.subList(0, i + 1), "lemma"); addNgram("lemma", ng); } } //empty ngram list and add remaining ngrams to the feature list while (!ngrams.isEmpty()) { String ng = featureFromArray(ngrams, "lemma"); addNgram("lemma", ng); ngrams.removeFirst(); } } return 1; }
From source file:com.quix.aia.cn.imo.mapper.UserMaintenance.java
/** * <p>This method retrieves USer lists based on search criteria</p> * @param req Servlet Request Parameter * @return User lists //from www . ja va 2 s. co m */ public Pager getUserListing(HttpServletRequest requestParamters) { // TODO Auto-generated method stub LinkedList item = new LinkedList(); User user = null; ArrayList vecAllRes = new ArrayList(); vecAllRes = getSearchUser(requestParamters); if (vecAllRes.size() != 0) { for (int i = 0; i < vecAllRes.size(); i++) { user = new User(); user = (User) vecAllRes.get(i); if (user.getCreatedUserSscCode() > 0) user.setCreatedUserSscLevel(true); else if (user.getCreatedUserCityCode() > 0) user.setCreatedUserCityLevel(true); else if (user.getCreatedUserDistCode() > 0) user.setCreatedUserDistLevel(true); else if (user.getCreatedUserBuCode() > 0) user.setCreatedUserBuLevel(true); item.add(user.getGetResListingTableRow(i, requestParamters, user)); } } Pager pager = new Pager(); pager.setActualSize(item.size()); pager.setCurrentPageNumber(0); pager.setMaxIndexPages(10); pager.setMaxPageItems(10); for (; item.size() % 10 != 0; item.add("<tr></tr>")) ; pager.setItems(item); return pager; }
From source file:org.epics.archiverappliance.retrieval.DataRetrievalServlet.java
private void doGetSinglePV(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { PoorMansProfiler pmansProfiler = new PoorMansProfiler(); String pvName = req.getParameter("pv"); if (configService.getStartupState() != STARTUP_SEQUENCE.STARTUP_COMPLETE) { String msg = "Cannot process data retrieval requests for PV " + pvName + " until the appliance has completely started up."; logger.error(msg);/*from www .j av a 2 s . com*/ resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, msg); return; } String startTimeStr = req.getParameter("from"); String endTimeStr = req.getParameter("to"); boolean useReduced = false; String useReducedStr = req.getParameter("usereduced"); if (useReducedStr != null && !useReducedStr.equals("")) { try { useReduced = Boolean.parseBoolean(useReducedStr); } catch (Exception ex) { logger.error("Exception parsing usereduced", ex); useReduced = false; } } String extension = req.getPathInfo().split("\\.")[1]; logger.info("Mime is " + extension); boolean useChunkedEncoding = true; String doNotChunkStr = req.getParameter("donotchunk"); if (doNotChunkStr != null && !doNotChunkStr.equals("false")) { logger.info("Turning off HTTP chunked encoding"); useChunkedEncoding = false; } boolean fetchLatestMetadata = false; String fetchLatestMetadataStr = req.getParameter("fetchLatestMetadata"); if (fetchLatestMetadataStr != null && fetchLatestMetadataStr.equals("true")) { logger.info("Adding a call to the engine to fetch the latest metadata"); fetchLatestMetadata = true; } // For data retrieval we need a PV info. However, in case of PV's that have long since retired, we may not want to have PVTypeInfo's in the system. // So, we support a template PV that lays out the data sources. // During retrieval, you can pass in the PV as a template and we'll clone this and make a temporary copy. String retiredPVTemplate = req.getParameter("retiredPVTemplate"); if (pvName == null) { String msg = "PV name is null."; resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_BAD_REQUEST, msg); return; } if (pvName.equals(ARCH_APPL_PING_PV)) { logger.debug("Processing ping PV - this is used to validate the connection with the client."); processPingPV(req, resp); return; } if (pvName.endsWith(".VAL")) { int len = pvName.length(); pvName = pvName.substring(0, len - 4); logger.info("Removing .VAL from pvName for request giving " + pvName); } // ISO datetimes are of the form "2011-02-02T08:00:00.000Z" Timestamp end = TimeUtils.plusHours(TimeUtils.now(), 1); if (endTimeStr != null) { try { end = TimeUtils.convertFromISO8601String(endTimeStr); } catch (IllegalArgumentException ex) { try { end = TimeUtils.convertFromDateTimeStringWithOffset(endTimeStr); } catch (IllegalArgumentException ex2) { String msg = "Cannot parse time" + endTimeStr; logger.warn(msg, ex2); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_BAD_REQUEST, msg); return; } } } // We get one day by default Timestamp start = TimeUtils.minusDays(end, 1); if (startTimeStr != null) { try { start = TimeUtils.convertFromISO8601String(startTimeStr); } catch (IllegalArgumentException ex) { try { start = TimeUtils.convertFromDateTimeStringWithOffset(startTimeStr); } catch (IllegalArgumentException ex2) { String msg = "Cannot parse time " + startTimeStr; logger.warn(msg, ex2); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_BAD_REQUEST, msg); return; } } } if (end.before(start)) { String msg = "For request, end " + end.toString() + " is before start " + start.toString() + " for pv " + pvName; logger.error(msg); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } LinkedList<TimeSpan> requestTimes = new LinkedList<TimeSpan>(); // We can specify a list of time stamp pairs using the optional timeranges parameter String timeRangesStr = req.getParameter("timeranges"); if (timeRangesStr != null) { boolean continueWithRequest = parseTimeRanges(resp, pvName, requestTimes, timeRangesStr); if (!continueWithRequest) { // Cannot parse the time ranges properly; we so abort the request. return; } // Override the start and the end so that the mergededup consumer works correctly. start = requestTimes.getFirst().getStartTime(); end = requestTimes.getLast().getEndTime(); } else { requestTimes.add(new TimeSpan(start, end)); } assert (requestTimes.size() > 0); String postProcessorUserArg = req.getParameter("pp"); if (pvName.contains("(")) { if (!pvName.contains(")")) { logger.error("Unbalanced paran " + pvName); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_BAD_REQUEST); return; } String[] components = pvName.split("[(,)]"); postProcessorUserArg = components[0]; pvName = components[1]; if (components.length > 2) { for (int i = 2; i < components.length; i++) { postProcessorUserArg = postProcessorUserArg + "_" + components[i]; } } logger.info("After parsing the function call syntax pvName is " + pvName + " and postProcessorUserArg is " + postProcessorUserArg); } PostProcessor postProcessor = PostProcessors.findPostProcessor(postProcessorUserArg); PVTypeInfo typeInfo = PVNames.determineAppropriatePVTypeInfo(pvName, configService); pmansProfiler.mark("After PVTypeInfo"); if (typeInfo == null && RetrievalState.includeExternalServers(req)) { logger.debug("Checking to see if pv " + pvName + " is served by a external Archiver Server"); typeInfo = checkIfPVisServedByExternalServer(pvName, start, req, resp, useChunkedEncoding); } if (typeInfo == null) { if (resp.isCommitted()) { logger.debug("Proxied the data thru an external server for PV " + pvName); return; } } if (typeInfo == null) { if (retiredPVTemplate != null) { PVTypeInfo templateTypeInfo = PVNames.determineAppropriatePVTypeInfo(retiredPVTemplate, configService); if (templateTypeInfo != null) { typeInfo = new PVTypeInfo(pvName, templateTypeInfo); typeInfo.setPaused(true); typeInfo.setApplianceIdentity(configService.getMyApplianceInfo().getIdentity()); // Somehow tell the code downstream that this is a fake typeInfo. typeInfo.setSamplingMethod(SamplingMethod.DONT_ARCHIVE); logger.debug("Using a template PV for " + pvName + " Need to determine the actual DBR type."); setActualDBRTypeFromData(pvName, typeInfo, configService); } } } if (typeInfo == null) { logger.error("Unable to find typeinfo for pv " + pvName); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_NOT_FOUND); return; } if (postProcessor == null) { if (useReduced) { String defaultPPClassName = configService.getInstallationProperties().getProperty( "org.epics.archiverappliance.retrieval.DefaultUseReducedPostProcessor", FirstSamplePP.class.getName()); logger.debug("Using the default usereduced preprocessor " + defaultPPClassName); try { postProcessor = (PostProcessor) Class.forName(defaultPPClassName).newInstance(); } catch (Exception ex) { logger.error("Exception constructing new instance of post processor " + defaultPPClassName, ex); postProcessor = null; } } } if (postProcessor == null) { logger.debug("Using the default raw preprocessor"); postProcessor = new DefaultRawPostProcessor(); } ApplianceInfo applianceForPV = configService.getApplianceForPV(pvName); if (applianceForPV == null) { // TypeInfo cannot be null here... assert (typeInfo != null); applianceForPV = configService.getAppliance(typeInfo.getApplianceIdentity()); } if (!applianceForPV.equals(configService.getMyApplianceInfo())) { // Data for pv is elsewhere. Proxy/redirect and return. proxyRetrievalRequest(req, resp, pvName, useChunkedEncoding, applianceForPV.getRetrievalURL() + "/../data"); return; } pmansProfiler.mark("After Appliance Info"); String pvNameFromRequest = pvName; String fieldName = PVNames.getFieldName(pvName); if (fieldName != null && !fieldName.equals("") && !pvName.equals(typeInfo.getPvName())) { logger.debug("We reset the pvName " + pvName + " to one from the typeinfo " + typeInfo.getPvName() + " as that determines the name of the stream. Also using ExtraFieldsPostProcessor"); pvName = typeInfo.getPvName(); postProcessor = new ExtraFieldsPostProcessor(fieldName); } try { // Postprocessors get their mandatory arguments from the request. // If user does not pass in the expected request, throw an exception. postProcessor.initialize(postProcessorUserArg, pvName); } catch (Exception ex) { logger.error("Postprocessor threw an exception during initialization for " + pvName, ex); resp.addHeader(MimeResponse.ACCESS_CONTROL_ALLOW_ORIGIN, "*"); resp.sendError(HttpServletResponse.SC_NOT_FOUND); return; } try (BasicContext retrievalContext = new BasicContext(typeInfo.getDBRType(), pvNameFromRequest); MergeDedupConsumer mergeDedupCountingConsumer = createMergeDedupConsumer(resp, extension, useChunkedEncoding); RetrievalExecutorResult executorResult = determineExecutorForPostProcessing(pvName, typeInfo, requestTimes, req, postProcessor)) { HashMap<String, String> engineMetadata = null; if (fetchLatestMetadata) { // Make a call to the engine to fetch the latest metadata. engineMetadata = fetchLatestMedataFromEngine(pvName, applianceForPV); } LinkedList<Future<RetrievalResult>> retrievalResultFutures = resolveAllDataSources(pvName, typeInfo, postProcessor, applianceForPV, retrievalContext, executorResult, req, resp); pmansProfiler.mark("After data source resolution"); long s1 = System.currentTimeMillis(); String currentlyProcessingPV = null; List<Future<EventStream>> eventStreamFutures = getEventStreamFuturesFromRetrievalResults(executorResult, retrievalResultFutures); logger.debug( "Done with the RetrievalResult's; moving onto the individual event stream from each source for " + pvName); pmansProfiler.mark("After retrieval results"); for (Future<EventStream> future : eventStreamFutures) { EventStreamDesc sourceDesc = null; try (EventStream eventStream = future.get()) { sourceDesc = null; // Reset it for each loop iteration. sourceDesc = eventStream.getDescription(); if (sourceDesc == null) { logger.warn("Skipping event stream without a desc for pv " + pvName); continue; } logger.debug("Processing event stream for pv " + pvName + " from source " + ((eventStream.getDescription() != null) ? eventStream.getDescription().getSource() : " unknown")); try { mergeTypeInfo(typeInfo, sourceDesc, engineMetadata); } catch (MismatchedDBRTypeException mex) { logger.error(mex.getMessage(), mex); continue; } if (currentlyProcessingPV == null || !currentlyProcessingPV.equals(pvName)) { logger.debug("Switching to new PV " + pvName + " In some mime responses we insert special headers at the beginning of the response. Calling the hook for that"); currentlyProcessingPV = pvName; mergeDedupCountingConsumer.processingPV(currentlyProcessingPV, start, end, (eventStream != null) ? sourceDesc : null); } try { // If the postProcessor does not have a consolidated event stream, we send each eventstream across as we encounter it. // Else we send the consolidatedEventStream down below. if (!(postProcessor instanceof PostProcessorWithConsolidatedEventStream)) { mergeDedupCountingConsumer.consumeEventStream(eventStream); resp.flushBuffer(); } } catch (Exception ex) { if (ex != null && ex.toString() != null && ex.toString().contains("ClientAbortException")) { // We check for ClientAbortException etc this way to avoid including tomcat jars in the build path. logger.debug( "Exception when consuming and flushing data from " + sourceDesc.getSource(), ex); } else { logger.error("Exception when consuming and flushing data from " + sourceDesc.getSource() + "-->" + ex.toString(), ex); } } pmansProfiler.mark("After event stream " + eventStream.getDescription().getSource()); } catch (Exception ex) { if (ex != null && ex.toString() != null && ex.toString().contains("ClientAbortException")) { // We check for ClientAbortException etc this way to avoid including tomcat jars in the build path. logger.debug("Exception when consuming and flushing data from " + (sourceDesc != null ? sourceDesc.getSource() : "N/A"), ex); } else { logger.error("Exception when consuming and flushing data from " + (sourceDesc != null ? sourceDesc.getSource() : "N/A") + "-->" + ex.toString(), ex); } } } if (postProcessor instanceof PostProcessorWithConsolidatedEventStream) { try (EventStream eventStream = ((PostProcessorWithConsolidatedEventStream) postProcessor) .getConsolidatedEventStream()) { EventStreamDesc sourceDesc = eventStream.getDescription(); if (sourceDesc == null) { logger.error("Skipping event stream without a desc for pv " + pvName + " and post processor " + postProcessor.getExtension()); } else { mergeDedupCountingConsumer.consumeEventStream(eventStream); resp.flushBuffer(); } } } // If the postProcessor needs to send final data across, give it a chance now... if (postProcessor instanceof AfterAllStreams) { EventStream finalEventStream = ((AfterAllStreams) postProcessor).anyFinalData(); if (finalEventStream != null) { mergeDedupCountingConsumer.consumeEventStream(finalEventStream); resp.flushBuffer(); } } pmansProfiler.mark("After writing all eventstreams to response"); long s2 = System.currentTimeMillis(); logger.info("For the complete request, found a total of " + mergeDedupCountingConsumer.totalEventsForAllPVs + " in " + (s2 - s1) + "(ms)" + " skipping " + mergeDedupCountingConsumer.skippedEventsForAllPVs + " events" + " deduping involved " + mergeDedupCountingConsumer.comparedEventsForAllPVs + " compares."); } catch (Exception ex) { if (ex != null && ex.toString() != null && ex.toString().contains("ClientAbortException")) { // We check for ClientAbortException etc this way to avoid including tomcat jars in the build path. logger.debug("Exception when retrieving data ", ex); } else { logger.error("Exception when retrieving data " + "-->" + ex.toString(), ex); } } pmansProfiler.mark("After all closes and flushing all buffers"); // Till we determine all the if conditions where we log this, we log sparingly.. if (pmansProfiler.totalTimeMS() > 5000) { logger.error("Retrieval time for " + pvName + " from " + startTimeStr + " to " + endTimeStr + pmansProfiler.toString()); } }
From source file:com.oltpbenchmark.benchmarks.seats.SEATSWorker.java
/** * Execute the FindOpenSeat procedure//from ww w.j a va2 s . c o m * @throws SQLException */ private boolean executeFindOpenSeats(FindOpenSeats proc) throws SQLException { final FlightId search_flight = this.profile.getRandomFlightId(); assert (search_flight != null); Long airport_depart_id = search_flight.getDepartAirportId(); if (LOG.isTraceEnabled()) LOG.trace("Calling " + proc); Object[][] results = proc.run(conn, search_flight.encode()); conn.commit(); int rowCount = results.length; assert (rowCount <= SEATSConstants.FLIGHTS_NUM_SEATS) : String .format("Unexpected %d open seats returned for %s", rowCount, search_flight); // there is some tiny probability of an empty flight .. maybe 1/(20**150) // if you hit this assert (with valid code), play the lottery! if (rowCount == 0) return (true); LinkedList<Reservation> cache = CACHE_RESERVATIONS.get(CacheType.PENDING_INSERTS); assert (cache != null) : "Unexpected " + CacheType.PENDING_INSERTS; // Store pending reservations in our queue for a later transaction BitSet seats = getSeatsBitSet(search_flight); tmp_reservations.clear(); for (Object row[] : results) { if (row == null) continue; // || rng.nextInt(100) < 75) continue; // HACK Integer seatnum = (Integer) row[1]; // We first try to get a CustomerId based at this departure airport if (LOG.isTraceEnabled()) LOG.trace("Looking for a random customer to fly on " + search_flight); CustomerId customer_id = profile.getRandomCustomerId(airport_depart_id); // We will go for a random one if: // (1) The Customer is already booked on this Flight // (2) We already made a new Reservation just now for this Customer int tries = SEATSConstants.FLIGHTS_NUM_SEATS; while (tries-- > 0 && (customer_id == null)) { // || isCustomerBookedOnFlight(customer_id, flight_id))) { customer_id = profile.getRandomCustomerId(); if (LOG.isTraceEnabled()) LOG.trace("RANDOM CUSTOMER: " + customer_id); } // WHILE assert (customer_id != null) : String.format( "Failed to find a unique Customer to reserve for seat #%d on %s", seatnum, search_flight); Reservation r = new Reservation(profile.getNextReservationId(getId()), search_flight, customer_id, seatnum.intValue()); seats.set(seatnum); tmp_reservations.add(r); if (LOG.isTraceEnabled()) LOG.trace( "QUEUED INSERT: " + search_flight + " / " + search_flight.encode() + " -> " + customer_id); } // WHILE if (tmp_reservations.isEmpty() == false) { Collections.shuffle(tmp_reservations); cache.addAll(tmp_reservations); while (cache.size() > SEATSConstants.CACHE_LIMIT_PENDING_INSERTS) { cache.remove(); } // WHILE if (LOG.isDebugEnabled()) LOG.debug(String.format("Stored %d pending inserts for %s [totalPendingInserts=%d]", tmp_reservations.size(), search_flight, cache.size())); } return (true); }
From source file:com.sun.portal.rssportlet.SettingsHandler.java
/** * Initialize the <code>RssPortletBean</code>. * * This is called after the bean has been set into this handler. *///from w w w. ja v a 2s .c om private void initSettingsBean() { PortletSession psession = getPortletRequest().getPortletSession(); //mandatory feeds mandate_feeds = (String[]) psession.getAttribute(MANDATE_FEEDS); //organizational feeds role_feeds = (String[]) psession.getAttribute(ROLE_FEEDS); if (role_feeds == null) { Map userInfo = (Map) portletRequest.getAttribute(PortletRequest.USER_INFO); String organizationalStatus = (String) userInfo.get("userRole"); String userRoleFeed = null; if ("Student".equalsIgnoreCase(organizationalStatus)) { userRoleFeed = STUDENT_FEEDS; } else if ("GradStudent".equalsIgnoreCase(organizationalStatus)) { userRoleFeed = GRAD_STUDENT_FEEDS; } else if ("Employee:Staff".equalsIgnoreCase(organizationalStatus)) { userRoleFeed = STAFF_FEEDS; } else if ("Employee:Faculty".equalsIgnoreCase(organizationalStatus)) { userRoleFeed = FACULTY_FEEDS; } else if ("Guest".equalsIgnoreCase(organizationalStatus)) { userRoleFeed = GUEST_FEEDS; } else { userRoleFeed = GUEST_FEEDS; } role_feeds = (String[]) psession.getAttribute(userRoleFeed); psession.setAttribute(ROLE_FEEDS, role_feeds); } //these are the feeds pushed at discrete time levels in the application //the level (horizon) of feeds that are pushed to the user and //seen by the user are tracked in user pref. attribute default_feeds_level default_feeds = (List) psession.getAttribute(DEFAULT_FEEDS); String[] sessionDefaultFeeds = null; if (null != default_feeds && default_feeds.size() != 0) { int default_feeds_level = 1; //every user gets some pushed feeds String feedLevel = (String) psession.getAttribute(SessionKeys.DEFAULT_FEEDS_LEVEL); if (null != feedLevel) { try { default_feeds_level = Integer.parseInt(feedLevel); } catch (NumberFormatException nfe) { default_feeds_level = 1; } } else { default_feeds_level = getIntPreference(PrefKeys.DEFAULT_FEEDS_LEVEL, 1); //stote in session and pref. psession.setAttribute(SessionKeys.DEFAULT_FEEDS_LEVEL, String.valueOf(default_feeds.size())); } sessionDefaultFeeds = (String[]) psession.getAttribute(SESSION_DEFAULT_FEEDS); if (null != sessionDefaultFeeds && sessionDefaultFeeds.length != 0) { sessionDefaultFeeds = new String[] {}; Iterator it = default_feeds.listIterator(); int counter = 1; int newDefaultFeeds = 0; while (it.hasNext()) { if (counter >= default_feeds_level) { String[] defaultFeeds = (String[]) it.next(); for (int i = 0; i < defaultFeeds.length; i++) { sessionDefaultFeeds[newDefaultFeeds] = defaultFeeds[i]; newDefaultFeeds++; } } counter++; } psession.setAttribute(SESSION_DEFAULT_FEEDS, sessionDefaultFeeds); } } //user selected feeds LinkedList feeds = new LinkedList( Arrays.asList(getPortletPreferences().getValues(PrefKeys.USER_FEEDS, new String[] {}))); feeds = stripDelimiter(feeds); //add mandatory+role feeds to feeds list if (null != role_feeds) { if (log.isDebugEnabled()) { log.debug("SetHandler feed role_feeds.length***********" + role_feeds.length); } for (int i = 0; i < role_feeds.length; i++) { feeds.add(i, role_feeds[i]); } } if (null != mandate_feeds) { if (log.isDebugEnabled()) { log.debug("SetHandler feed mandate_feeds.length***********" + mandate_feeds.length); } for (int i = 0; i < mandate_feeds.length; i++) { feeds.add(i, mandate_feeds[i]); } } if (null != sessionDefaultFeeds && sessionDefaultFeeds.length != 0) { if (log.isDebugEnabled()) { log.debug("SetHandler feed mandate_feeds.length***********" + sessionDefaultFeeds.length); } for (int i = 0; i < sessionDefaultFeeds.length; i++) { feeds.add(i, sessionDefaultFeeds[i]); } } if (log.isDebugEnabled()) { Iterator it = feeds.listIterator(); while (it.hasNext()) { log.debug("SettingsHandler feed LinkedList***********" + (String) it.next()); } } getSettingsBean().setFeeds(feeds); int maxAge = getIntPreference(PrefKeys.MAX_AGE, 72); getSettingsBean().setMaxAge(maxAge); int cacheTimeout = getIntPreference(PrefKeys.CACHE_TIMEOUT, 3600); getSettingsBean().setCacheTimeout(cacheTimeout); int maxDescriptionLength = getIntPreference(PrefKeys.MAX_DESCRIPTION_LENGTH, 512); getSettingsBean().setMaxDescriptionLength(maxDescriptionLength); boolean newWindow = getBooleanPreference(PrefKeys.NEWWIN, false); getSettingsBean().setNewWindow(newWindow); boolean disableMaxAge = getBooleanPreference(PrefKeys.DISABLE_MAX_AGE, false); getSettingsBean().setDisableMaxAge(disableMaxAge); int maxEntries = getIntPreference(PrefKeys.MAX_ENTRIES, 5); getSettingsBean().setMaxEntries(maxEntries); if (feeds != null && feeds.size() != 0) { String startFeed = getPortletPreferences().getValue(PrefKeys.START_FEED, (String) feeds.get(0)); System.out.println("SettingsHandler PrefKeys.START_FEED ***********" + startFeed); if (startFeed != null) { startFeed = stripDelimiter(startFeed); } getSettingsBean().setStartFeed(startFeed); } initSelectedFeed(); getSettingsBean().setLocale(getPortletRequest().getLocale()); //set feed titles if (feeds != null && feeds.size() != 0) { setFeedTitleList(feeds); } }
From source file:com.datatorrent.stram.client.EventsAgent.java
public List<EventInfo> getLatestEvents(String appId, int limit) { LinkedList<EventInfo> result = new LinkedList<EventInfo>(); String dir = getEventsDirectory(appId); if (dir == null) { return null; }/*from w w w.j ava 2 s. c om*/ long totalNumEvents = 0; IndexFileBufferedReader ifbr = null; LinkedList<Pair<String, Long>> partFiles = new LinkedList<Pair<String, Long>>(); try { ifbr = new IndexFileBufferedReader(new InputStreamReader( stramAgent.getFileSystem().open(new Path(dir, FSPartFileCollection.INDEX_FILE))), dir); EventsIndexLine indexLine; while ((indexLine = (EventsIndexLine) ifbr.readIndexLine()) != null) { if (indexLine.isEndLine) { continue; } partFiles.add(new Pair<String, Long>(indexLine.partFile, indexLine.numEvents)); totalNumEvents += indexLine.numEvents; } } catch (Exception ex) { LOG.warn("Got exception when reading events", ex); return result; } finally { IOUtils.closeQuietly(ifbr); } long offset = 0; while (totalNumEvents > limit && !partFiles.isEmpty()) { Pair<String, Long> head = partFiles.getFirst(); if (totalNumEvents - head.second < limit) { offset = Math.max(0, totalNumEvents - limit); break; } totalNumEvents -= head.second; partFiles.removeFirst(); } String lastProcessPartFile = null; for (Pair<String, Long> partFile : partFiles) { BufferedReader partBr = null; try { partBr = new BufferedReader( new InputStreamReader(stramAgent.getFileSystem().open(new Path(dir, partFile.first)))); processPartFile(partBr, null, null, offset, limit, result); offset = 0; lastProcessPartFile = partFile.first; } catch (Exception ex) { LOG.warn("Got exception when reading events", ex); } finally { IOUtils.closeQuietly(partBr); } } BufferedReader partBr = null; try { String extraPartFile = getNextPartFile(lastProcessPartFile); if (extraPartFile != null && limit > 0) { partBr = new BufferedReader( new InputStreamReader(stramAgent.getFileSystem().open(new Path(dir, extraPartFile)))); processPartFile(partBr, null, null, 0, Integer.MAX_VALUE, result); } } catch (Exception ex) { // ignore } finally { IOUtils.closeQuietly(partBr); } while (result.size() > limit) { result.removeFirst(); } return result; }
From source file:com.moviejukebox.plugin.KinopoiskPlugin.java
/** * Scan Kinopoisk html page for the specified movie *//* w w w . ja va 2 s . c o m*/ private boolean updateKinopoiskMediaInfo(Movie movie, String kinopoiskId) { try { String originalTitle = movie.getTitle(); String newTitle = originalTitle; String xml = httpClient.request(FILM_URL + kinopoiskId, CHARSET); boolean etalonFlag = kinopoiskId.equals(etalonId); // Work-around for issue #649 xml = xml.replace("…", "…"); xml = xml.replace("—", "—"); // Title boolean overrideTitle = OverrideTools.checkOverwriteTitle(movie, KINOPOISK_PLUGIN_ID); if (overrideTitle || etalonFlag) { newTitle = HTMLTools.extractTag(xml, "class=\"moviename-big\" itemprop=\"name\">", 0, "</"); if (StringTools.isValidString(newTitle)) { if (overrideTitle) { int i = newTitle.indexOf("(?"); if (i >= 0) { newTitle = newTitle.substring(0, i); movie.setMovieType(Movie.TYPE_TVSHOW); } newTitle = newTitle.replace('\u00A0', ' ').trim(); if (movie.getSeason() != -1) { newTitle = newTitle + ", ? " + movie.getSeason(); } // Original title originalTitle = newTitle; for (String s : HTMLTools.extractTags(xml, "class=\"moviename-big\" itemprop=\"name\">", "</span>", "itemprop=\"alternativeHeadline\">", "</span>")) { if (!s.isEmpty()) { originalTitle = s; if (joinTitles) { newTitle = newTitle + Movie.SPACE_SLASH_SPACE + originalTitle; } } break; } } else { newTitle = originalTitle; } } else { if (etalonFlag) { LOG.error(SITE_DESIGN, "movie title"); } newTitle = originalTitle; } } // Plot if (OverrideTools.checkOverwritePlot(movie, KINOPOISK_PLUGIN_ID)) { StringBuilder plot = new StringBuilder(); for (String subPlot : HTMLTools.extractTags(xml, "<span class=\"_reachbanner_\"", "</span>", "", "<")) { if (!subPlot.isEmpty()) { if (plot.length() > 0) { plot.append(" "); } plot.append(subPlot); } } movie.setPlot(plot.toString(), KINOPOISK_PLUGIN_ID); if (etalonFlag && (plot.length() == 0)) { LOG.error(SITE_DESIGN, "plot"); } } boolean valueFounded = false; boolean genresFounded = false; boolean certificationFounded = false; boolean countryFounded = false; boolean yearFounded = false; boolean runtimeFounded = false; boolean taglineFounded = false; boolean releaseFounded = false; for (String item : HTMLTools.extractTags(xml, "<table class=\"info\">", "</table>", "<tr", "</tr>")) { item = "<tr>" + item + "</tr>"; // Genres if (OverrideTools.checkOverwriteGenres(movie, KINOPOISK_PLUGIN_ID)) { LinkedList<String> newGenres = new LinkedList<>(); boolean innerGenresFound = false; for (String genre : HTMLTools.extractTags(item, "><", "</tr>", "<a href=\"/lists/m_act%5Bgenre%5D/", "</a>")) { innerGenresFound = true; genre = genre.substring(0, 1).toUpperCase() + genre.substring(1, genre.length()); if ("".equalsIgnoreCase(genre)) { newGenres.addFirst(genre); } else { newGenres.add(genre); } } if (innerGenresFound) { // Limit genres count int maxGenres = 9; try { maxGenres = PropertiesUtil.getIntProperty("genres.max", 9); } catch (Exception ignore) { // } while (newGenres.size() > maxGenres) { newGenres.removeLast(); } movie.setGenres(newGenres, KINOPOISK_PLUGIN_ID); genresFounded = true; } } // Certification from MPAA if (OverrideTools.checkOverwriteCertification(movie, KINOPOISK_PLUGIN_ID)) { if (!certificationFounded) { certificationFounded = getCertification(movie, item, "> MPAA<", "</tr>", "<a href=\"/film", "</a>", "alt=\" "); } if (!certificationFounded || russianCertification) { certificationFounded |= getCertification(movie, item, ">?<", "</tr>", "<td style=\"color", "</span>", "ageLimit "); } } // Country if (OverrideTools.checkOverwriteCountry(movie, KINOPOISK_PLUGIN_ID)) { Collection<String> scraped = HTMLTools.extractTags(item, ">?<", "</tr>", "a href=\"/lists/m_act%5Bcountry%5D/", "</a>"); if (scraped != null && !scraped.isEmpty()) { List<String> countries = new ArrayList<>(); for (String country : scraped) { if (translitCountry) { country = FileTools.makeSafeFilename(country); } countries.add(country); if (!countryAll) { // just first country, so break here break; } } movie.setCountries(countries, KINOPOISK_PLUGIN_ID); countryFounded = true; } } // Year if (OverrideTools.checkOverwriteYear(movie, KINOPOISK_PLUGIN_ID)) { for (String year : HTMLTools.extractTags(item, "><", "</tr>", "<a href=\"/lists/m_act%5Byear%5D/", "</a>")) { movie.setYear(year, KINOPOISK_PLUGIN_ID); yearFounded = true; break; } } // Run time if (OverrideTools.checkOverwriteRuntime(movie, KINOPOISK_PLUGIN_ID) || etalonFlag) { for (String runtime : HTMLTools.extractTags(item, ">?<", "</tr>", "<td", "</td>")) { if (runtime.contains("<span")) { runtime = runtime.substring(0, runtime.indexOf("<span")); } movie.setRuntime(runtime, KINOPOISK_PLUGIN_ID); runtimeFounded = true; break; } } // Tagline if (OverrideTools.checkOverwriteTagline(movie, KINOPOISK_PLUGIN_ID)) { for (String tagline : HTMLTools.extractTags(item, ">?<", "</tr>", "<td ", "</td>")) { if (tagline.length() > 0) { movie.setTagline(tagline.replace("\u00AB", "\"").replace("\u00BB", "\""), KINOPOISK_PLUGIN_ID); taglineFounded = true; break; } } } // Release date if (OverrideTools.checkOverwriteReleaseDate(movie, KINOPOISK_PLUGIN_ID)) { String releaseDate = ""; for (String release : HTMLTools.extractTags(item, "> ()<", "</tr>", "<a href=\"/film/", "</a>")) { releaseDate = release; releaseFounded = true; break; } if (StringUtils.isBlank(releaseDate)) { for (String release : HTMLTools.extractTags(item, "> ()<", "</tr>", "<a href=\"/film/", "</a>")) { releaseDate = release; releaseFounded = true; break; } } movie.setReleaseDate(releaseDate, KINOPOISK_PLUGIN_ID); } } if (etalonFlag) { if (!genresFounded) { LOG.error(SITE_DESIGN, "genres"); } if (!certificationFounded) { LOG.error(SITE_DESIGN, "certification"); } if (!countryFounded) { LOG.error(SITE_DESIGN, "country"); } if (!yearFounded) { LOG.error(SITE_DESIGN, "year"); } if (!runtimeFounded) { LOG.error(SITE_DESIGN, "runtime"); } if (!taglineFounded) { LOG.error(SITE_DESIGN, "tagline"); } if (!releaseFounded) { LOG.error(SITE_DESIGN, "release"); } } // Rating if (!nfoRating) { valueFounded = false; for (String rating : HTMLTools.extractTags(xml, "<a href=\"/film/" + kinopoiskId + "/votes/\"", "</a>", "<span", "</span>")) { int kinopoiskRating = StringTools.parseRating(rating); movie.addRating(KINOPOISK_PLUGIN_ID, kinopoiskRating); valueFounded = true; break; } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "rating"); } int imdbRating = movie.getRating(IMDB_PLUGIN_ID); if (imdbRating == -1 || etalonFlag) { // Get IMDB rating from kinopoisk page String rating = HTMLTools.extractTag(xml, ">IMDb:", 0, " ("); valueFounded = false; if (StringTools.isValidString(rating)) { imdbRating = StringTools.parseRating(rating); movie.addRating(IMDB_PLUGIN_ID, imdbRating); valueFounded = true; } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "IMDB rating"); } } } // Top250 if (OverrideTools.checkOverwriteTop250(movie, KINOPOISK_PLUGIN_ID)) { movie.setTop250(HTMLTools.extractTag(xml, "<a href=\"/level/20/#", 0, "\""), KINOPOISK_PLUGIN_ID); } // Poster String posterURL = movie.getPosterURL(); if (StringTools.isNotValidString(posterURL) || (!nfoPoster && poster) || etalonFlag) { valueFounded = false; if (poster || etalonFlag) { String previousURL = posterURL; posterURL = Movie.UNKNOWN; // Load page with all poster String wholeArts = httpClient.request(FILM_URL + kinopoiskId + "/posters/", CHARSET); if (StringTools.isValidString(wholeArts)) { if (wholeArts.contains("<table class=\"fotos")) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/poster/sm_", 0, "\""); if (StringTools.isValidString(picture)) { posterURL = "http://st.kinopoisk.ru/images/poster/" + picture; valueFounded = true; } } } if (StringTools.isNotValidString(posterURL)) { posterURL = previousURL; } if (StringTools.isValidString(posterURL)) { movie.setPosterURL(posterURL); movie.setPosterFilename(movie.getBaseName() + ".jpg"); LOG.debug("Set poster URL to {} for {}", posterURL, movie.getBaseName()); } } if (StringTools.isNotValidString(movie.getPosterURL())) { if (overrideTitle) { movie.setTitle(originalTitle, KINOPOISK_PLUGIN_ID); } // Removing Poster info from plugins. Use of PosterScanner routine instead. // movie.setPosterURL(locatePosterURL(movie, "")); } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "poster"); } } // Fanart String fanURL = movie.getFanartURL(); if (StringTools.isNotValidString(fanURL) || (!nfoFanart && (fanArt || kadr)) || etalonFlag || fanURL.contains("originalnull")) { valueFounded = false; try { String previousURL = fanURL; fanURL = Movie.UNKNOWN; // Load page with all wallpaper String wholeArts = httpClient.request("http://www.kinopoisk.ru/film/" + kinopoiskId + "/wall/", CHARSET); if (StringTools.isValidString(wholeArts)) { if (wholeArts.contains("<table class=\"fotos")) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/wallpaper/sm_", 0, ".jpg"); if (StringTools.isValidString(picture)) { if (picture.contains("_")) { picture = picture.substring(0, picture.indexOf('_')); } String size = HTMLTools.extractTag(wholeArts, "<u><a href=\"/picture/" + picture + "/w_size/", 0, "/"); wholeArts = httpClient.request( "http://www.kinopoisk.ru/picture/" + picture + "/w_size/" + size, CHARSET); if (StringTools.isValidString(wholeArts)) { picture = HTMLTools.extractTag(wholeArts, "src=\"http://st-im.kinopoisk.r/im/wallpaper/", 0, "\""); if (StringTools.isValidString(picture)) { fanURL = "http://st.kinopoisk.ru/im/wallpaper/" + picture; valueFounded = true; } } } } } if (kadr && (StringTools.isNotValidString(fanURL) || fanURL.contains("originalnull"))) { // Load page with all videoimage wholeArts = httpClient.request(FILM_URL + kinopoiskId + "/stills/", CHARSET); if (StringTools.isValidString(wholeArts)) { // Looking for photos table int photosInd = wholeArts.indexOf("<table class=\"fotos"); if (photosInd != -1) { String picture = HTMLTools.extractTag(wholeArts, "src=\"http://st.kinopoisk.ru/images/kadr/sm_", 0, "\""); if (StringTools.isValidString(picture)) { fanURL = "http://www.kinopoisk.ru/images/kadr/" + picture; valueFounded = true; } } } } if (StringTools.isNotValidString(fanURL)) { fanURL = previousURL; } if (StringTools.isValidString(fanURL)) { movie.setFanartURL(fanURL); movie.setFanartFilename(movie.getBaseName() + fanartToken + "." + fanartExtension); LOG.debug("Set fanart URL to {} for {}", fanURL, movie.getBaseName()); } } catch (IOException ex) { // Ignore } } // Studio/Company if (OverrideTools.checkOverwriteCompany(movie, KINOPOISK_PLUGIN_ID)) { xml = httpClient.request(FILM_URL + kinopoiskId + "/studio/", CHARSET); valueFounded = false; if (StringTools.isValidString(xml)) { Collection<String> studio = new ArrayList<>(); if (xml.contains(">?:<")) { for (String tmp : HTMLTools.extractTags(xml, ">?:<", "</table>", "<a href=\"/lists/m_act%5Bstudio%5D/", "</a>")) { studio.add(HTMLTools.removeHtmlTags(tmp)); if (!companyAll) { break; } } } if (xml.contains(">:<") && (companyAll || studio.isEmpty())) { for (String tmp : HTMLTools.extractTags(xml, ">:<", "</table>", "<a href=\"/lists/m_act%5Bcompany%5D/", "</a>")) { studio.add(HTMLTools.removeHtmlTags(tmp)); } } if (!studio.isEmpty()) { movie.setCompany(companyAll ? StringUtils.join(studio, Movie.SPACE_SLASH_SPACE) : new ArrayList<>(studio).get(0), KINOPOISK_PLUGIN_ID); valueFounded = true; } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "company"); } } // Awards if ((scrapeAwards && !nfoAwards) || etalonFlag) { if (clearAward) { movie.clearAwards(); } xml = httpClient.request("http://www.kinopoisk.ru/film/" + kinopoiskId + "/awards/", CHARSET); Collection<AwardEvent> awards = new ArrayList<>(); if (StringTools.isValidString(xml)) { int beginIndex = xml.indexOf("<b><a href=\"/awards/"); if (beginIndex != -1) { for (String item : HTMLTools.extractTags(xml, "<table cellspacing=0 cellpadding=0 border=0 width=100%>", "<br /><br /><br /><br /><br /><br />", "<table cellspacing=\"0\" cellpadding=\"0\" border=\"0\" width=\"100%\" ", "</table>")) { String name = Movie.UNKNOWN; int year = -1; int won = 0; int nominated = 0; Collection<String> wons = new ArrayList<>(); Collection<String> nominations = new ArrayList<>(); for (String tmp : HTMLTools.extractTags(item, "<td height=\"40\" class=\"news\" style=\"padding: 10px\">", "</td>", "<a href=\"/awards/", "</a>")) { int coma = tmp.indexOf(","); name = tmp.substring(0, coma); year = NumberUtils.toInt(tmp.substring(coma + 2, coma + 6)); break; } for (String tmp : HTMLTools.extractTags(item, "><", ":", "(", ")")) { won = NumberUtils.toInt(tmp); break; } if (won > 0) { for (String tmp : HTMLTools.extractTags(item, "><", "</ul>", "<li class=\"trivia\">", "</li>")) { wons.add(HTMLTools.removeHtmlTags(tmp).replaceAll(" {2,}", " ")); } } for (String tmp : HTMLTools.extractTags(item, ">?<", ":", "(", ")")) { nominated = NumberUtils.toInt(tmp); break; } if (nominated > 0) { for (String tmp : HTMLTools.extractTags(item, ">?<", "</ul>", "<li class=\"trivia\"", "</li>")) { nominations.add(HTMLTools.removeHtmlTags(tmp).replaceAll(" {2,}", " ")); } } if (StringTools.isValidString(name) && year > 1900 && year < 2020 && (!scrapeWonAwards || (won > 0))) { Award award = new Award(); award.setName(name); award.setYear(year); if (won > 0) { award.setWons(wons); } if (nominated > 0) { award.setNominations(nominations); } AwardEvent event = new AwardEvent(); event.setName(name); event.addAward(award); awards.add(event); } } } } if (!awards.isEmpty()) { movie.setAwards(awards); } else if (etalonFlag) { LOG.error(SITE_DESIGN, "award"); } } // Cast enhancement boolean overrideCast = OverrideTools.checkOverwriteActors(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleCast = OverrideTools.checkOverwritePeopleActors(movie, KINOPOISK_PLUGIN_ID); boolean overrideDirectors = OverrideTools.checkOverwriteDirectors(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleDirectors = OverrideTools.checkOverwritePeopleDirectors(movie, KINOPOISK_PLUGIN_ID); boolean overrideWriters = OverrideTools.checkOverwriteWriters(movie, KINOPOISK_PLUGIN_ID); boolean overridePeopleWriters = OverrideTools.checkOverwritePeopleWriters(movie, KINOPOISK_PLUGIN_ID); if (overrideCast || overridePeopleCast || overrideDirectors || overridePeopleDirectors || overrideWriters || overridePeopleWriters || etalonFlag) { xml = httpClient.request(FILM_URL + kinopoiskId + "/cast", CHARSET); if (StringTools.isValidString(xml)) { if (overrideDirectors || overridePeopleDirectors || etalonFlag) { int count = scanMoviePerson(movie, xml, "director", directorMax, overrideDirectors, overridePeopleDirectors); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "directors"); } } if (overrideWriters || overridePeopleWriters || etalonFlag) { int count = scanMoviePerson(movie, xml, "writer", writerMax, overrideWriters, overridePeopleWriters); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "writers"); } } if (overrideCast || overridePeopleCast || etalonFlag) { int count = scanMoviePerson(movie, xml, "actor", actorMax, overrideCast, overridePeopleCast); if (etalonFlag && count == 0) { LOG.error(SITE_DESIGN, "cast"); } } Collection<Filmography> outcast = new ArrayList<>(); for (Filmography p : movie.getPeople()) { if (StringTools.isNotValidString(p.getId(KINOPOISK_PLUGIN_ID))) { outcast.add(p); } } for (Filmography p : outcast) { movie.removePerson(p); } } } // Business if (scrapeBusiness || etalonFlag) { xml = httpClient.request(FILM_URL + kinopoiskId + "/box/", CHARSET); if (StringTools.isValidString(xml)) { valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, ">:<", "</table>", "<font color=\"#ff6600\"", "</h3>")) { if (StringTools.isValidString(tmp)) { tmp = tmp.replaceAll("\u00A0", ",").replaceAll(",$", ""); if (StringTools.isValidString(tmp) && !"--".equals(tmp)) { movie.setBudget(tmp); valueFounded = true; break; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: summary"); } valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, "> -? (?)<", "</table>", "<h3 style=\"font-size: 18px; margin: 0; padding: 0;color:#f60\"", "</h3>")) { if (StringTools.isValidString(tmp)) { tmp = tmp.replaceAll("\u00A0", ",").replaceAll(",$", ""); if (StringTools.isValidString(tmp) && !"--".equals(tmp)) { movie.setOpenWeek("USA", tmp); valueFounded = true; break; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: first weekend"); } valueFounded = false; for (String tmp : HTMLTools.extractTags(xml, ">?? ?<", "</table>", "<tr><td colspan=2", "</h3>")) { tmp += "</h3>"; String country = HTMLTools.extractTag(tmp, ">", ":"); if (StringTools.isValidString(country)) { String money = HTMLTools .removeHtmlTags("<h3 " + HTMLTools.extractTag(tmp, "<h3 ", "</h3>")) .replaceAll("\u00A0", ",").replaceAll(",$", ""); if (!"--".equals(money)) { movie.setGross( " ??" .equals(country) ? "Russia" : " ?".equals(country) ? "USA" : " ?".equals(country) ? "Worldwide" : " ?".equals( country) ? "Others" : country, money); valueFounded = true; } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "business: gross"); } } else if (etalonFlag) { LOG.error(SITE_DESIGN, "business"); } } // Did You Know if (scrapeTrivia || etalonFlag) { if (clearTrivia) { movie.clearDidYouKnow(); } if (etalonFlag && triviaMax == 0) { triviaMax = 1; } if (triviaMax != 0) { xml = httpClient.request(FILM_URL + kinopoiskId + "/view_info/ok/#trivia", CHARSET); if (StringTools.isValidString(xml)) { int i = 0; for (String tmp : HTMLTools.extractTags(xml, "> , ...<", "</ul>", "<li class=\"trivia", "</li>")) { if (i < triviaMax || triviaMax == -1) { movie.addDidYouKnow(HTMLTools.removeHtmlTags(tmp)); valueFounded = true; i++; } else { break; } } } } if (!valueFounded && etalonFlag) { LOG.error(SITE_DESIGN, "trivia"); } } // Finally set title if (overrideTitle) { movie.setTitle(newTitle, KINOPOISK_PLUGIN_ID); } } catch (IOException | NumberFormatException error) { LOG.error("Failed retreiving movie data from Kinopoisk : {}", kinopoiskId); LOG.error(SystemTools.getStackTrace(error)); } return true; }
From source file:ddf.catalog.transformer.input.pdf.GeoPdfParserImpl.java
/** * Generates a WKT compliant String from a PDF Document if it contains GeoPDF information. * Currently, only WGS84 Projections are supported (GEOGRAPHIC GeoPDF ProjectionType). * * @param pdfDocument - The PDF document * @return the WKT String//from w w w .jav a2s .c o m * @throws IOException */ @Override public String apply(PDDocument pdfDocument) throws IOException { ToDoubleVisitor toDoubleVisitor = new ToDoubleVisitor(); LinkedList<String> polygons = new LinkedList<>(); for (PDPage pdPage : pdfDocument.getPages()) { COSDictionary cosObject = pdPage.getCOSObject(); COSBase lgiDictObject = cosObject.getObjectFromPath(LGIDICT); // Handle Multiple Map Frames if (lgiDictObject instanceof COSArray) { for (int i = 0; i < ((COSArray) lgiDictObject).size(); i++) { COSDictionary lgidict = (COSDictionary) cosObject.getObjectFromPath(LGIDICT + "/[" + i + "]"); COSDictionary projectionArray = (COSDictionary) lgidict.getDictionaryObject(PROJECTION); if (projectionArray != null) { String projectionType = ((COSString) projectionArray.getItem(PROJECTION_TYPE)).getString(); if (GEOGRAPHIC.equals(projectionType)) { COSArray neatlineArray = (COSArray) cosObject .getObjectFromPath(LGIDICT + "/[" + i + "]/" + NEATLINE); getWktFromNeatLine(lgidict, neatlineArray, toDoubleVisitor).ifPresent(polygons::add); } else { LOGGER.debug("Unsupported projection type {}. Map Frame will be skipped.", projectionType); } } else { LOGGER.debug("No projection array found on the map frame. Map Frame will be skipped."); } } // Handle One Map Frame } else if (lgiDictObject instanceof COSDictionary) { COSDictionary lgidict = (COSDictionary) lgiDictObject; COSDictionary projectionArray = (COSDictionary) lgidict.getDictionaryObject(PROJECTION); if (projectionArray != null) { String projectionType = ((COSString) projectionArray.getItem(PROJECTION_TYPE)).getString(); if (GEOGRAPHIC.equals(projectionType)) { COSArray neatlineArray = (COSArray) cosObject.getObjectFromPath(LGIDICT + "/" + NEATLINE); if (neatlineArray == null) { neatlineArray = generateNeatLineFromPDFDimensions(pdPage); } getWktFromNeatLine(lgidict, neatlineArray, toDoubleVisitor).ifPresent(polygons::add); } else { LOGGER.debug("Unsupported projection type {}. Map Frame will be skipped.", projectionType); } } else { LOGGER.debug("No projection array found on the map frame. Map Frame will be skipped."); } } } if (polygons.size() == 0) { LOGGER.debug( "No GeoPDF information found on PDF during transformation. Metacard location will not be set."); return null; } if (polygons.size() == 1) { return POLYGON + polygons.get(0) + "))"; } else { return polygons.stream().map(polygon -> "((" + polygon + "))") .collect(Collectors.joining(",", MULTIPOLYGON, ")")); } }