List of usage examples for org.joda.time DateTime getMillis
public long getMillis()
From source file:com.thinkbiganalytics.metadata.jpa.jobrepo.job.JpaBatchJobExecutionProvider.java
License:Apache License
public List<? extends BatchJobExecution> findLatestFinishedJobForFeedSince(String feedName, DateTime dateTime) { List<JpaBatchJobExecution> jobExecutions = null; if (dateTime == null) { jobExecutions = jobExecutionRepository.findLatestFinishedJobForFeed(feedName); } else {//from w ww . jav a 2 s . com jobExecutions = jobExecutionRepository.findLatestFinishedJobsForFeedSince(feedName, dateTime.getMillis()); } if (jobExecutions != null) { return jobExecutions; } else { return Collections.emptyList(); } }
From source file:com.thinkbiganalytics.metadata.jpa.jobrepo.nifi.JpaNifiFeedProcessorStats.java
License:Apache License
@Override public void setMinEventTime(DateTime minEventTime) { this.minEventTime = minEventTime; setMinEventTimeMillis(minEventTime.getMillis()); }
From source file:com.thinkbiganalytics.metadata.jpa.support.GenericQueryDslFilter.java
License:Apache License
private static Long convertDateTimeStringToMillis(String filterKey, Object value) { Long millis = null;//from www . j a v a 2s . c om if (CommonFilterTranslations.isDateStoredAsMillisField(filterKey) && value instanceof String) { //attempt to convert the incoming value to a long try { DateTime dateTime = Formatters.parseDateTime((String) value); millis = dateTime.getMillis(); } catch (IllegalArgumentException e) { //uanble to parse the value } } return millis; }
From source file:com.thinkbiganalytics.metadata.rest.model.nifi.NiFiFlowCacheSync.java
License:Apache License
public boolean needsUpdate(DateTime lastUpdated) { return (lastUpdated == null || snapshot == null || (snapshot.getSnapshotDate() == null) || (snapshot.getSnapshotDate() != null && lastUpdated.getMillis() != snapshot.getSnapshotDate().getMillis())); }
From source file:com.thinkbiganalytics.metadata.sla.api.core.FeedFailureMetricAssessor.java
License:Apache License
@Override public void assess(FeedFailedMetric metric, MetricAssessmentBuilder<Serializable> builder) { builder.metric(metric);// w w w . ja va 2 s . c o m String feedName = metric.getFeedName(); HashMap<String, String> data = new HashMap<>(); data.put("feed", feedName); FeedFailureService.LastFeedJob lastFeedJob = feedFailureService.findLatestJob(feedName); DateTime lastTime = lastFeedJob.getDateTime(); data.put("dateTime", lastTime.toString()); data.put("dateTimeMillis", lastTime.getMillis() + ""); LOG.debug("Assessing FeedFailureMetric for '{}'. The Last Feed Job was: {} ", feedName, lastFeedJob); if (!feedFailureService.isEmptyJob(lastFeedJob)) { if (lastFeedJob.getBatchJobExecutionId() != null) { data.put("jobExecutionId", lastFeedJob.getBatchJobExecutionId().toString()); } //compare with the latest feed time, alerts with same timestamps will not be raised builder.compareWith(feedName, lastTime.getMillis()); if (lastFeedJob.isFailure()) { data.put("status", "FAILURE"); String message = "Feed " + feedName + " has failed on " + lastFeedJob.getDateTime(); if (lastFeedJob.getBatchJobExecutionId() != null) { message += ". Batch Job ExecutionId: " + lastFeedJob.getBatchJobExecutionId(); } LOG.debug(message); builder.message(message).data(data).result(AssessmentResult.FAILURE); } else { data.put("status", "SUCCESS"); String message = "Feed " + feedName + " has succeeded on " + lastFeedJob.getDateTime(); if (lastFeedJob.getBatchJobExecutionId() != null) { message += ". Batch Job ExecutionId: " + lastFeedJob.getBatchJobExecutionId(); } LOG.debug(message); builder.message(message).data(data).result(AssessmentResult.SUCCESS); } } else { LOG.debug("FeedFailureMetric found an no recent jobs for '{}'. Returning SUCCESS ", feedName); builder.data(data).message("No Jobs found for feed " + feedName + " since " + lastFeedJob.getDateTime()) .result(AssessmentResult.SUCCESS); } }
From source file:com.thinkbiganalytics.metadata.sla.api.core.FeedOnTimeArrivalMetricAssessor.java
License:Apache License
@Override @SuppressWarnings("unchecked") public void assess(FeedOnTimeArrivalMetric metric, MetricAssessmentBuilder builder) { LOG.debug("Assessing metric: ", metric); builder.metric(metric);/*from w w w. ja v a 2 s.c o m*/ String feedName = metric.getFeedName(); DateTime lastFeedTime = feedProvider.getLastActiveTimeStamp(feedName); HashMap<String, String> data = new HashMap<>(); data.put("feed", feedName); Long nowDiff = 0L; Period nowDiffPeriod = new Period(nowDiff.longValue()); if (lastFeedTime != null) { nowDiff = DateTime.now().getMillis() - lastFeedTime.getMillis(); nowDiffPeriod = new Period(nowDiff.longValue()); } Long latePeriodMillis = metric.getLatePeriod().toStandardDuration().getMillis(); Long duration = CronExpressionUtil.getCronInterval(metric.getExpectedExpression()); Period acceptedPeriod = new Period(duration + latePeriodMillis); Date expectedDate = CronExpressionUtil.getPreviousFireTime(metric.getExpectedExpression()); DateTime expectedTime = new DateTime(expectedDate); LOG.debug("Calculated the Expected Date to be {} ", expectedTime); DateTime lateTime = expectedTime.plus(metric.getLatePeriod()); LOG.debug("CurrentTime is: {}. Comparing {} against the lateTime of {} ", DateTime.now(), lastFeedTime, lateTime); builder.compareWith(expectedDate, feedName); data.put("expectedTime", expectedTime.toString()); data.put("expectedTimeMillis", expectedTime.getMillis() + ""); data.put("lateTime", expectedTime.toString()); data.put("lateTimeMillis", expectedTime.getMillis() + ""); if (lastFeedTime != null) { data.put("lastFeedTime", lastFeedTime.toString()); data.put("lastFeedTimeMillis", lastFeedTime.getMillis() + ""); } builder.data(data); if (lastFeedTime == null) { LOG.debug("Feed with the specified name {} not found", feedName); builder.message("Feed with the specified name " + feedName + " not found ") .result(AssessmentResult.WARNING); } else if (lastFeedTime.isAfter(expectedTime) && lastFeedTime.isBefore(lateTime)) { LOG.debug("Data for feed {} arrived on {}, which was before late time: {}", feedName, lastFeedTime, lateTime); builder.message("Data for feed " + feedName + " arrived on " + lastFeedTime + ", which was before late time: " + lateTime).result(AssessmentResult.SUCCESS); } else if (lastFeedTime.isAfter(lateTime)) { LOG.debug("Data for feed {} has not arrived before the late time: {} ", feedName, lateTime); builder.message("Data for feed " + feedName + " has not arrived before the late time: " + lateTime + "\n The last successful feed was on " + lastFeedTime).result(AssessmentResult.FAILURE); } else if (nowDiff <= (duration + latePeriodMillis)) { LOG.debug( "Data for feed {} has arrived before the late time: {}. The last successful feed was on {}. It has been {} since data has arrived. The allowed duration is {} ", feedName, lateTime, lastFeedTime, DateTimeUtil.formatPeriod(nowDiffPeriod), DateTimeUtil.formatPeriod(acceptedPeriod)); builder.message( "Data for feed " + feedName + " has arrived on time. \n The last successful feed was on " + lastFeedTime + ". It has been " + DateTimeUtil.formatPeriod(nowDiffPeriod) + " since data has arrived. The allowed duration is " + DateTimeUtil.formatPeriod(acceptedPeriod)) .result(AssessmentResult.SUCCESS); } else if (nowDiff > (duration + latePeriodMillis)) { //error its been greater that the duration of the cron + lateTime LOG.debug( "Data for feed {} has not arrived before the late time: {}. The last successful feed was on {}. It has been {} since data has arrived. The allowed duration is {} ", feedName, lateTime, lastFeedTime, DateTimeUtil.formatPeriod(nowDiffPeriod), DateTimeUtil.formatPeriod(acceptedPeriod)); builder.message( "Data for feed " + feedName + " has not arrived on time. \n The last successful feed was on " + lastFeedTime + ". It has been " + DateTimeUtil.formatPeriod(nowDiffPeriod) + " since data has arrived. The allowed duration is " + DateTimeUtil.formatPeriod(acceptedPeriod)) .result(AssessmentResult.FAILURE); } else if (DateTime.now().isBefore(lateTime)) { //&& lastFeedTime.isBefore(expectedTime) LOG.debug("CurrentTime {} is before the lateTime of {}. Not Assessing", DateTime.now(), lateTime); return; } else { LOG.debug("Data for feed {} has not arrived before the late time: {} ", feedName, lateTime); builder.message("Data for feed " + feedName + " has not arrived before the late time: " + lateTime + "\n The last successful feed was on " + lastFeedTime).result(AssessmentResult.FAILURE); } }
From source file:com.thinkbiganalytics.nifi.provenance.reporting.KyloProvenanceEventReportingTask.java
License:Apache License
/** * * Responsible to querying the provenance data and sending the events to Kylo, both the streaming event aggregration and the batch event data A boolean {@code processing} flag is used to prevent * multiple threads from running this trigger at the same time. 1. sets the Boolean flag to processing 2. queries NiFi provenance to determine the set of Events to process and send to Kylo 3. * aggregrates and processes the batch events and sends to Kylo via JMS 4. Callback listeners for the JMS will update the {@code StateManager} setting the {@code LAST_EVENT_ID_KEY} value. 5. Upon * any failure the {@code abortProcessing()} will be called *//* w ww. ja v a 2s .c om*/ @Override public void onTrigger(final ReportingContext context) { String nodeId = getNodeIdStrategy().getNodeId(context); getLogger().debug("Nifi nodeId {}", new Object[] { nodeId }); if (nodeId == null) { return; } ensureInitializeFlowFileMapDbCache(); if (!isInitializing() && processing.compareAndSet(false, true)) { final StateManager stateManager = context.getStateManager(); final EventAccess access = context.getEventAccess(); final ProvenanceEventRepository provenance = access.getProvenanceRepository(); if (this.stateManager == null) { this.stateManager = stateManager; } getLogger().debug( "KyloProvenanceEventReportingTask onTrigger Info: Reporting Task Triggered! Last Event id is ", new Object[] { getLastEventId(stateManager) }); ensureJmsListeners(); //get the latest event Id in provenance final Long maxEventId = provenance.getMaxEventId(); if (maxEventId == null || maxEventId < 0) { getLogger().debug("No Provenance exists yet. Max Id is not set.. Will not process events "); finishProcessing(0); return; } previousMax = maxEventId; try { if (!isKyloAvailable()) { getLogger().info( "Kylo is not available to process requests yet. This task will exit and wait for its next schedule interval."); abortProcessing(); return; } //get the last event that was processed long lastEventId = initializeAndGetLastEventIdForProcessing(maxEventId, nodeId); //finish processing if there is nothing to process if (lastEventId == maxEventId.longValue()) { getLogger().trace("Last event id == max id... will not process!"); finishProcessing(0); return; } DateTime lastLogTime = DateTime.now(); //how often to report the batch processing log when processing a lot of events int logReportingTimeMs = 10000; //every 10 sec long nextId = lastEventId + 1; //record count is inclusive, so we need to add one to the difference to include the last eventid int recordCount = new Long(maxEventId - (nextId < 0 ? 0 : nextId)).intValue() + 1; int totalRecords = recordCount; long start = System.currentTimeMillis(); //split this into batches of events, maxing at 500 if not specified int batchSize = processingBatchSize == null || processingBatchSize < 1 ? 500 : processingBatchSize; //setup the object pool to be able to store at least the processing batch size amount ProvenanceEventObjectPool pool = getProvenanceEventObjectPool(); int total = processingBatchSize + 100; pool.setMaxIdle(total); pool.setMaxTotal(total); Integer batches = (int) Math.ceil(Double.valueOf(recordCount) / batchSize); if (recordCount > 0) { getLogger().info( "KyloProvenanceEventReportingTask onTrigger Info: KyloFlowCache Sync Id: {} . Attempting to process {} events starting with event id: {}. Splitting into {} batches of {} each ", new Object[] { nifiFlowSyncId, recordCount, nextId, batches, batchSize }); } //reset the queryTime holder nifiQueryTime = 0L; while (recordCount > 0) { if (!isProcessing()) { break; } long min = lastEventId + 1; long max = (min + (batchSize - 1)) > maxEventId ? maxEventId : (min + (batchSize - 1)); int batchAmount = new Long(max - (min < 0 ? 0 : min)).intValue() + 1; if (batchAmount <= 0) { break; } else { lastEventId = processEventsInRange(provenance, min, max); recordCount -= batchAmount; recordCount = recordCount < 0 ? 0 : recordCount; setLastEventId(lastEventId); if (lastLogTime == null || (DateTime.now().getMillis() - lastLogTime.getMillis() > logReportingTimeMs)) { lastLogTime = DateTime.now(); getLogger().info( "KyloProvenanceEventReportingTask onTrigger Info: ReportingTask is in a long running process. Currently processing Event id: {}. {} events remaining to be processed. ", new Object[] { lastEventId, recordCount }); } } if (!isProcessing()) { break; } } if (totalRecords > 0 && isProcessing()) { long processingTime = (System.currentTimeMillis() - start); getLogger().info( "KyloProvenanceEventReportingTask onTrigger Info: ReportingTask finished. Last Event id: {}. Total time to process {} events was {} ms. Total time spent querying for events in Nifi was {} ms. Kylo ProcessingTime: {} ms ", new Object[] { lastEventId, totalRecords, processingTime, nifiQueryTime, processingTime - nifiQueryTime }); } finishProcessing(totalRecords); } catch (IOException e) { getLogger().error(e.getMessage(), e); } finally { abortProcessing(); } } else { if (isInitializing()) { getLogger().info( "Still initializing any previously active flow file provenance data. The task should run shortly"); } else { Long maxId = context.getEventAccess().getProvenanceRepository().getMaxEventId(); Long count = (maxId - previousMax); getLogger().info("KyloProvenanceEventReportingTask onTrigger Info: Still processing previous batch " + currentProcessingMessage + ". The next run will process events up to " + maxId + ". " + count + " new events"); } } }
From source file:com.threewks.thundr.transformer.date.DateTimeToBigDecimal.java
License:Apache License
@Override public BigDecimal from(DateTime from) { return from == null ? null : new BigDecimal(from.getMillis()); }
From source file:com.threewks.thundr.transformer.date.DateTimeToLong.java
License:Apache License
@Override public Long from(DateTime from) { return from == null ? null : from.getMillis(); }
From source file:com.threewks.thundr.user.gae.User.java
License:Apache License
@Override public void setLastLogin(DateTime lastLogin) { this.lastLogin = lastLogin.getMillis(); }