List of usage examples for org.apache.hadoop.conf Configuration getLong
public long getLong(String name, long defaultValue)
name property as a long. From source file:org.apache.nutch.searcher.DistributedSearchBean.java
License:Apache License
public DistributedSearchBean(Configuration conf, Path luceneConfig, Path solrConfig) throws IOException { FileSystem fs = FileSystem.get(conf); this.timeout = conf.getLong("ipc.client.timeout", 60000); List<SearchBean> beanList = new ArrayList<SearchBean>(); if (fs.exists(luceneConfig)) { LOG.info("Adding Nutch searchers in " + luceneConfig.makeQualified(fs).toUri()); addLuceneBeans(beanList, luceneConfig, conf); }/*from ww w. j a v a2 s . c o m*/ if (fs.exists(solrConfig)) { LOG.info("Adding Solr searchers in " + solrConfig.makeQualified(fs).toUri()); addSolrBeans(beanList, solrConfig, conf); } LOG.info("Added " + beanList.size() + " remote searchers."); beans = beanList.toArray(new SearchBean[beanList.size()]); liveServers = new boolean[beans.length]; for (int i = 0; i < liveServers.length; i++) { liveServers[i] = true; } searchTasks = new ArrayList<Callable<Hits>>(); detailTasks = new ArrayList<Callable<HitDetails[]>>(); pingWorkers = new ArrayList<PingWorker>(); for (int i = 0; i < beans.length; i++) { searchTasks.add(new SearchTask(i)); detailTasks.add(new DetailTask(i)); pingWorkers.add(new PingWorker(i)); } pingService = Executors.newScheduledThreadPool(beans.length); for (PingWorker worker : pingWorkers) { pingService.scheduleAtFixedRate(worker, 0, 10, TimeUnit.SECONDS); } }
From source file:org.apache.nutch.searcher.DistributedSegmentBean.java
License:Apache License
public DistributedSegmentBean(Configuration conf, Path serversConfig) throws IOException { this.timeout = conf.getLong("ipc.client.timeout", 60000); List<SegmentBean> beanList = new ArrayList<SegmentBean>(); List<InetSocketAddress> segmentServers = NutchBean.readAddresses(serversConfig, conf); for (InetSocketAddress addr : segmentServers) { SegmentBean bean = (RPCSegmentBean) RPC.getProxy(RPCSegmentBean.class, FetchedSegments.VERSION, addr, conf);//ww w . ja v a 2 s.c o m beanList.add(bean); } beans = beanList.toArray(new SegmentBean[beanList.size()]); summaryTasks = new ArrayList<Callable<Summary[]>>(beans.length); segmentWorkers = new ArrayList<SegmentWorker>(beans.length); for (int i = 0; i < beans.length; i++) { summaryTasks.add(new DistSummmaryTask(i)); segmentWorkers.add(new SegmentWorker(i)); } segmentMap = new ConcurrentHashMap<String, Integer>(); pingService = Executors.newScheduledThreadPool(beans.length); for (SegmentWorker worker : segmentWorkers) { pingService.scheduleAtFixedRate(worker, 0, 30, TimeUnit.SECONDS); } }
From source file:org.apache.nutch.segment.SegmentMerger.java
License:Apache License
public void setConf(Configuration conf) { super.setConf(conf); if (conf == null) return;//from w w w .j a v a 2s .c o m if (conf.getBoolean("segment.merger.filter", false)) { filters = new URLFilters(conf); mergeFilters = new SegmentMergeFilters(conf); } if (conf.getBoolean("segment.merger.normalizer", false)) normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_DEFAULT); sliceSize = conf.getLong("segment.merger.slice", -1); if ((sliceSize > 0) && (LOG.isInfoEnabled())) { LOG.info("Slice size: " + sliceSize + " URLs."); } }
From source file:org.apache.oozie.command.wf.ActionEndCommand.java
License:Open Source License
@Override protected Void call(WorkflowStore store) throws StoreException, CommandException { WorkflowJobBean workflow = store.getWorkflow(jobId, false); setLogInfo(workflow);//from ww w .j a v a 2 s .c o m WorkflowActionBean action = store.getAction(id, false); setLogInfo(action); if (action.isPending() && (action.getStatus() == WorkflowActionBean.Status.DONE || action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL)) { if (workflow.getStatus() == WorkflowJob.Status.RUNNING) { ActionExecutor executor = Services.get().get(ActionService.class).getExecutor(action.getType()); Configuration conf = workflow.getWorkflowInstance().getConf(); int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); long retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); if (executor != null) { boolean isRetry = false; if (action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) { isRetry = true; } ActionExecutorContext context = new ActionCommand.ActionExecutorContext(workflow, action, isRetry); try { XLog.getLog(getClass()).debug( "End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]", action.getName(), action.getType(), action.getStatus(), action.getExternalStatus(), action.getSignalValue()); WorkflowInstance wfInstance = workflow.getWorkflowInstance(); DagELFunctions.setActionInfo(wfInstance, action); workflow.setWorkflowInstance(wfInstance); incrActionCounter(action.getType(), 1); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); executor.end(context, action); cron.stop(); addActionCron(action.getType(), cron); if (!context.isEnded()) { XLog.getLog(getClass()).warn(XLog.OPS, "Action Ended, ActionExecutor [{0}] must call setEndData()", executor.getType()); action.setErrorInfo(END_DATA_MISSING, "Execution Ended, but End Data Missing from Action"); failJob(context); store.updateAction(action); store.updateWorkflow(workflow); return null; } action.setRetries(0); action.setEndTime(new Date()); store.updateAction(action); store.updateWorkflow(workflow); Status slaStatus = null; switch (action.getStatus()) { case OK: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; break; case ERROR: XLog.getLog(getClass()).info("ERROR is considered as FAILED for SLA"); slaStatus = Status.KILLED; break; default: // TODO: What will happen for other Action // status slaStatus = Status.FAILED; break; } SLADbOperations.writeStausEvent(action.getSlaXml(), action.getId(), store, slaStatus, SlaAppType.WORKFLOW_ACTION); queueCallable(new NotificationCommand(workflow, action)); XLog.getLog(getClass()).debug("Queuing commands for action=" + id + ", status=" + action.getStatus() + ", Set pending=" + action.getPending()); queueCallable(new SignalCommand(workflow.getId(), id)); } catch (ActionExecutorException ex) { XLog.getLog(getClass()).warn( "Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage()); action.setErrorInfo(ex.getErrorCode(), ex.getMessage()); action.setEndTime(null); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) { handleNonTransient(store, context, executor, WorkflowAction.Status.END_MANUAL); action.setPendingAge(new Date()); action.setRetries(0); } action.setEndTime(null); break; case NON_TRANSIENT: handleNonTransient(store, context, executor, WorkflowAction.Status.END_MANUAL); action.setEndTime(null); break; case ERROR: handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR); queueCallable(new SignalCommand(workflow.getId(), id)); break; case FAILED: failJob(context); break; } store.updateAction(action); store.updateWorkflow(workflow); } } else { throw new CommandException(ErrorCode.E0802, action.getType()); } } else { XLog.getLog(getClass()).warn("Job state is not {0}. Skipping ActionEnd Execution", WorkflowJob.Status.RUNNING.toString()); } } else { XLog.getLog(getClass()).debug("Action pending={0}, status={1}. Skipping ActionEnd Execution", action.getPending(), action.getStatusStr()); } return null; }
From source file:org.apache.oozie.command.wf.ActionEndXCommand.java
License:Apache License
@Override protected Void execute() throws CommandException { LOG.debug("STARTED ActionEndXCommand for action " + actionId); Configuration conf = wfJob.getWorkflowInstance().getConf(); int maxRetries = 0; long retryInterval = 0; if (!(executor instanceof ControlNodeActionExecutor)) { maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); }//from ww w .j a va2 s .co m executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); boolean isRetry = false; if (wfAction.getStatus() == WorkflowActionBean.Status.END_RETRY || wfAction.getStatus() == WorkflowActionBean.Status.END_MANUAL) { isRetry = true; } boolean isUserRetry = false; ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction, isRetry, isUserRetry); try { LOG.debug("End, name [{0}] type [{1}] status[{2}] external status [{3}] signal value [{4}]", wfAction.getName(), wfAction.getType(), wfAction.getStatus(), wfAction.getExternalStatus(), wfAction.getSignalValue()); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); executor.end(context, wfAction); cron.stop(); addActionCron(wfAction.getType(), cron); incrActionCounter(wfAction.getType(), 1); if (!context.isEnded()) { LOG.warn(XLog.OPS, "Action Ended, ActionExecutor [{0}] must call setEndData()", executor.getType()); wfAction.setErrorInfo(END_DATA_MISSING, "Execution Ended, but End Data Missing from Action"); failJob(context); } else { wfAction.setRetries(0); wfAction.setEndTime(new Date()); boolean shouldHandleUserRetry = false; Status slaStatus = null; switch (wfAction.getStatus()) { case OK: slaStatus = Status.SUCCEEDED; break; case KILLED: slaStatus = Status.KILLED; break; case FAILED: slaStatus = Status.FAILED; shouldHandleUserRetry = true; break; case ERROR: LOG.info("ERROR is considered as FAILED for SLA"); slaStatus = Status.KILLED; shouldHandleUserRetry = true; break; default: slaStatus = Status.FAILED; shouldHandleUserRetry = true; break; } if (!shouldHandleUserRetry || !handleUserRetry(wfAction)) { SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), slaStatus, SlaAppType.WORKFLOW_ACTION); if (slaEvent != null) { insertList.add(slaEvent); } } } WorkflowInstance wfInstance = wfJob.getWorkflowInstance(); DagELFunctions.setActionInfo(wfInstance, wfAction); wfJob.setWorkflowInstance(wfInstance); updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_END, wfAction)); wfJob.setLastModifiedTime(new Date()); updateList.add(new UpdateEntry<WorkflowJobQuery>( WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob)); } catch (ActionExecutorException ex) { LOG.warn("Error ending action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", wfAction.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage()); wfAction.setErrorInfo(ex.getErrorCode(), ex.getMessage()); wfAction.setEndTime(null); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.END_RETRY)) { handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); wfAction.setPendingAge(new Date()); wfAction.setRetries(0); } wfAction.setEndTime(null); break; case NON_TRANSIENT: handleNonTransient(context, executor, WorkflowAction.Status.END_MANUAL); wfAction.setEndTime(null); break; case ERROR: handleError(context, executor, COULD_NOT_END, false, WorkflowAction.Status.ERROR); break; case FAILED: failJob(context); break; } WorkflowInstance wfInstance = wfJob.getWorkflowInstance(); DagELFunctions.setActionInfo(wfInstance, wfAction); wfJob.setWorkflowInstance(wfInstance); updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_END, wfAction)); wfJob.setLastModifiedTime(new Date()); updateList.add(new UpdateEntry<WorkflowJobQuery>( WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob)); } finally { try { BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null); } catch (JPAExecutorException e) { throw new CommandException(e); } if (!(executor instanceof ControlNodeActionExecutor) && EventHandlerService.isEnabled()) { generateEvent(wfAction, wfJob.getUser()); } new SignalXCommand(jobId, actionId).call(); } LOG.debug("ENDED ActionEndXCommand for action " + actionId); return null; }
From source file:org.apache.oozie.command.wf.ActionStartCommand.java
License:Open Source License
@Override protected Void call(WorkflowStore store) throws StoreException, CommandException { WorkflowJobBean workflow = store.getWorkflow(jobId, false); setLogInfo(workflow);/*from ww w . java 2s . c o m*/ WorkflowActionBean action = store.getAction(id, false); XLog.getLog(getClass()).warn(XLog.STD, "[***" + action.getId() + "***]" + "In call()....status=" + action.getStatusStr()); setLogInfo(action); if (action.isPending() && (action.getStatus() == WorkflowActionBean.Status.PREP || action.getStatus() == WorkflowActionBean.Status.START_RETRY || action.getStatus() == WorkflowActionBean.Status.START_MANUAL)) { if (workflow.getStatus() == WorkflowJob.Status.RUNNING) { ActionExecutor executor = Services.get().get(ActionService.class).getExecutor(action.getType()); Configuration conf = workflow.getWorkflowInstance().getConf(); int maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); long retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); if (executor != null) { ActionExecutorContext context = null; try { boolean isRetry = false; if (action.getStatus() == WorkflowActionBean.Status.START_RETRY || action.getStatus() == WorkflowActionBean.Status.START_MANUAL) { isRetry = true; } context = new ActionCommand.ActionExecutorContext(workflow, action, isRetry); try { String tmpActionConf = XmlUtils.removeComments(action.getConf()); String actionConf = context.getELEvaluator().evaluate(tmpActionConf, String.class); action.setConf(actionConf); XLog.getLog(getClass()).debug("Start, name [{0}] type [{1}] configuration{E}{E}{2}{E}", action.getName(), action.getType(), actionConf); } catch (ELEvaluationException ex) { throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, EL_EVAL_ERROR, ex.getMessage(), ex); } catch (ELException ex) { context.setErrorInfo(EL_ERROR, ex.getMessage()); XLog.getLog(getClass()).warn("ELException in ActionStartCommand ", ex.getMessage(), ex); handleError(context, store, workflow, action); return null; } catch (org.jdom.JDOMException je) { context.setErrorInfo("ParsingError", je.getMessage()); XLog.getLog(getClass()).warn("JDOMException in ActionStartCommand ", je.getMessage(), je); handleError(context, store, workflow, action); return null; } catch (Exception ex) { context.setErrorInfo(EL_ERROR, ex.getMessage()); XLog.getLog(getClass()).warn("Exception in ActionStartCommand ", ex.getMessage(), ex); handleError(context, store, workflow, action); return null; } action.setErrorInfo(null, null); incrActionCounter(action.getType(), 1); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); executor.start(context, action); cron.stop(); FaultInjection.activate("org.apache.oozie.command.SkipCommitFaultInjection"); addActionCron(action.getType(), cron); action.setRetries(0); if (action.isExecutionComplete()) { if (!context.isExecuted()) { XLog.getLog(getClass()).warn(XLog.OPS, "Action Completed, ActionExecutor [{0}] must call setExecutionData()", executor.getType()); action.setErrorInfo(EXEC_DATA_MISSING, "Execution Complete, but Execution Data Missing from Action"); failJob(context); store.updateAction(action); store.updateWorkflow(workflow); return null; } action.setPending(); queueCallable(new ActionEndCommand(action.getId(), action.getType())); } else { if (!context.isStarted()) { XLog.getLog(getClass()).warn(XLog.OPS, "Action Started, ActionExecutor [{0}] must call setStartData()", executor.getType()); action.setErrorInfo(START_DATA_MISSING, "Execution Started, but Start Data Missing from Action"); failJob(context); store.updateAction(action); store.updateWorkflow(workflow); return null; } queueCallable(new NotificationCommand(workflow, action)); } XLog.getLog(getClass()).warn(XLog.STD, "[***" + action.getId() + "***]" + "Action status=" + action.getStatusStr()); store.updateAction(action); store.updateWorkflow(workflow); // Add SLA status event (STARTED) for WF_ACTION // SLADbOperations.writeSlaStatusEvent(eSla, // action.getId(), Status.STARTED, store); SLADbOperations.writeStausEvent(action.getSlaXml(), action.getId(), store, Status.STARTED, SlaAppType.WORKFLOW_ACTION); XLog.getLog(getClass()).warn(XLog.STD, "[***" + action.getId() + "***]" + "Action updated in DB!"); } catch (ActionExecutorException ex) { XLog.getLog(getClass()).warn( "Error starting action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", action.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage(), ex); action.setErrorInfo(ex.getErrorCode(), ex.getMessage()); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.START_RETRY)) { handleNonTransient(store, context, executor, WorkflowAction.Status.START_MANUAL); action.setPendingAge(new Date()); action.setRetries(0); action.setStartTime(null); } break; case NON_TRANSIENT: handleNonTransient(store, context, executor, WorkflowAction.Status.START_MANUAL); break; case ERROR: handleError(context, executor, WorkflowAction.Status.ERROR.toString(), true, WorkflowAction.Status.DONE); break; case FAILED: try { failJob(context); queueCallable(new CoordActionUpdateCommand(workflow)); SLADbOperations.writeStausEvent(action.getSlaXml(), action.getId(), store, Status.FAILED, SlaAppType.WORKFLOW_ACTION); SLADbOperations.writeStausEvent(workflow.getSlaXml(), workflow.getId(), store, Status.FAILED, SlaAppType.WORKFLOW_JOB); } catch (XException x) { XLog.getLog(getClass()).warn("ActionStartCommand - case:FAILED ", x.getMessage()); } break; } store.updateAction(action); store.updateWorkflow(workflow); } } else { throw new CommandException(ErrorCode.E0802, action.getType()); } } else { XLog.getLog(getClass()).warn("Job state is not {0}. Skipping Action Execution", WorkflowJob.Status.RUNNING.toString()); } } return null; }
From source file:org.apache.oozie.command.wf.ActionStartXCommand.java
License:Apache License
@Override protected ActionExecutorContext execute() throws CommandException { LOG.debug("STARTED ActionStartXCommand for wf actionId=" + actionId); Configuration conf = wfJob.getWorkflowInstance().getConf(); int maxRetries = 0; long retryInterval = 0; boolean execSynchronous = false; if (!(executor instanceof ControlNodeActionExecutor)) { maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries()); retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval()); }/*from w w w .ja va2 s . c o m*/ executor.setMaxRetries(maxRetries); executor.setRetryInterval(retryInterval); try { boolean isRetry = false; if (wfAction.getStatus() == WorkflowActionBean.Status.START_RETRY || wfAction.getStatus() == WorkflowActionBean.Status.START_MANUAL) { isRetry = true; prepareForRetry(wfAction); } boolean isUserRetry = false; if (wfAction.getStatus() == WorkflowActionBean.Status.USER_RETRY) { isUserRetry = true; prepareForRetry(wfAction); } context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction, isRetry, isUserRetry); boolean caught = false; try { if (!(executor instanceof ControlNodeActionExecutor)) { String tmpActionConf = XmlUtils.removeComments(wfAction.getConf()); String actionConf = context.getELEvaluator().evaluate(tmpActionConf, String.class); wfAction.setConf(actionConf); LOG.debug("Start, name [{0}] type [{1}] configuration{E}{E}{2}{E}", wfAction.getName(), wfAction.getType(), actionConf); } } catch (ELEvaluationException ex) { caught = true; throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, EL_EVAL_ERROR, ex.getMessage(), ex); } catch (ELException ex) { caught = true; context.setErrorInfo(EL_ERROR, ex.getMessage()); LOG.warn("ELException in ActionStartXCommand ", ex.getMessage(), ex); handleError(context, wfJob, wfAction); } catch (org.jdom.JDOMException je) { caught = true; context.setErrorInfo("ParsingError", je.getMessage()); LOG.warn("JDOMException in ActionStartXCommand ", je.getMessage(), je); handleError(context, wfJob, wfAction); } catch (Exception ex) { caught = true; context.setErrorInfo(EL_ERROR, ex.getMessage()); LOG.warn("Exception in ActionStartXCommand ", ex.getMessage(), ex); handleError(context, wfJob, wfAction); } if (!caught) { wfAction.setErrorInfo(null, null); incrActionCounter(wfAction.getType(), 1); LOG.info( "Start action [{0}] with user-retry state : userRetryCount [{1}], userRetryMax [{2}], userRetryInterval [{3}]", wfAction.getId(), wfAction.getUserRetryCount(), wfAction.getUserRetryMax(), wfAction.getUserRetryInterval()); Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); context.setStartTime(); /* Creating and forwarding the tag, It will be useful during repeat attempts of Launcher, to ensure only one child job is running. Tag is formed as follows: For workflow job, tag = action-id For Coord job, tag = coord-action-id@action-name (if not part of sub flow), else coord-action-id@subflow-action-name@action-name. */ if (conf.get(OOZIE_ACTION_YARN_TAG) != null) { context.setVar(OOZIE_ACTION_YARN_TAG, conf.get(OOZIE_ACTION_YARN_TAG) + "@" + wfAction.getName()); } else if (wfJob.getParentId() != null) { context.setVar(OOZIE_ACTION_YARN_TAG, wfJob.getParentId() + "@" + wfAction.getName()); } else { context.setVar(OOZIE_ACTION_YARN_TAG, wfAction.getId()); } executor.start(context, wfAction); cron.stop(); FaultInjection.activate("org.apache.oozie.command.SkipCommitFaultInjection"); addActionCron(wfAction.getType(), cron); wfAction.setRetries(0); if (wfAction.isExecutionComplete()) { if (!context.isExecuted()) { LOG.warn(XLog.OPS, "Action Completed, ActionExecutor [{0}] must call setExecutionData()", executor.getType()); wfAction.setErrorInfo(EXEC_DATA_MISSING, "Execution Complete, but Execution Data Missing from Action"); failJob(context); } else { wfAction.setPending(); if (!(executor instanceof ControlNodeActionExecutor)) { queue(new ActionEndXCommand(wfAction.getId(), wfAction.getType())); } else { execSynchronous = true; } } } else { if (!context.isStarted()) { LOG.warn(XLog.OPS, "Action Started, ActionExecutor [{0}] must call setStartData()", executor.getType()); wfAction.setErrorInfo(START_DATA_MISSING, "Execution Started, but Start Data Missing from Action"); failJob(context); } else { queue(new WorkflowNotificationXCommand(wfJob, wfAction)); } } LOG.info(XLog.STD, "[***" + wfAction.getId() + "***]" + "Action status=" + wfAction.getStatusStr()); updateList.add( new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, wfAction)); updateJobLastModified(); // Add SLA status event (STARTED) for WF_ACTION SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.STARTED, SlaAppType.WORKFLOW_ACTION); if (slaEvent != null) { insertList.add(slaEvent); } LOG.info(XLog.STD, "[***" + wfAction.getId() + "***]" + "Action updated in DB!"); } } catch (ActionExecutorException ex) { LOG.warn("Error starting action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", wfAction.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage(), ex); wfAction.setErrorInfo(ex.getErrorCode(), ex.getMessage()); switch (ex.getErrorType()) { case TRANSIENT: if (!handleTransient(context, executor, WorkflowAction.Status.START_RETRY)) { handleNonTransient(context, executor, WorkflowAction.Status.START_MANUAL); wfAction.setPendingAge(new Date()); wfAction.setRetries(0); wfAction.setStartTime(null); } break; case NON_TRANSIENT: handleNonTransient(context, executor, WorkflowAction.Status.START_MANUAL); break; case ERROR: handleError(context, executor, WorkflowAction.Status.ERROR.toString(), true, WorkflowAction.Status.DONE); break; case FAILED: try { failJob(context); endWF(); SLAEventBean slaEvent1 = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION); if (slaEvent1 != null) { insertList.add(slaEvent1); } } catch (XException x) { LOG.warn("ActionStartXCommand - case:FAILED ", x.getMessage()); } break; } updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, wfAction)); updateJobLastModified(); } finally { try { BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null); if (!(executor instanceof ControlNodeActionExecutor) && EventHandlerService.isEnabled()) { generateEvent(wfAction, wfJob.getUser()); } if (execSynchronous) { // Changing to synchronous call from asynchronous queuing to prevent // undue delay from ::start:: to action due to queuing callActionEnd(); } } catch (JPAExecutorException e) { throw new CommandException(e); } } LOG.debug("ENDED ActionStartXCommand for wf actionId=" + actionId + ", jobId=" + jobId); return null; }
From source file:org.apache.oozie.coord.CoordUtils.java
License:Apache License
/** * Check if sla alert is disabled for action. * @param actionBean//from w ww. jav a2s.c om * @param coordName * @param jobConf * @return * @throws ParseException */ public static boolean isSlaAlertDisabled(CoordinatorActionBean actionBean, String coordName, Configuration jobConf) throws ParseException { int disableSlaNotificationOlderThan = jobConf.getInt(OozieClient.SLA_DISABLE_ALERT_OLDER_THAN, ConfigurationService.getInt(OozieClient.SLA_DISABLE_ALERT_OLDER_THAN)); if (disableSlaNotificationOlderThan > 0) { // Disable alert for catchup jobs long timeDiffinHrs = TimeUnit.MILLISECONDS .toHours(new Date().getTime() - actionBean.getNominalTime().getTime()); if (timeDiffinHrs > jobConf.getLong(OozieClient.SLA_DISABLE_ALERT_OLDER_THAN, ConfigurationService.getLong(OozieClient.SLA_DISABLE_ALERT_OLDER_THAN))) { return true; } } boolean disableAlert = false; if (jobConf.get(OozieClient.SLA_DISABLE_ALERT_COORD) != null) { String coords = jobConf.get(OozieClient.SLA_DISABLE_ALERT_COORD); Set<String> coordsToDisableFor = new HashSet<String>(Arrays.asList(coords.split(","))); if (coordsToDisableFor.contains(coordName)) { return true; } if (coordsToDisableFor.contains(actionBean.getJobId())) { return true; } } // Check if sla alert is disabled for that action if (!StringUtils.isEmpty(jobConf.get(OozieClient.SLA_DISABLE_ALERT)) && getCoordActionSLAAlertStatus(actionBean, coordName, jobConf, OozieClient.SLA_DISABLE_ALERT)) { return true; } // Check if sla alert is enabled for that action if (!StringUtils.isEmpty(jobConf.get(OozieClient.SLA_ENABLE_ALERT)) && getCoordActionSLAAlertStatus(actionBean, coordName, jobConf, OozieClient.SLA_ENABLE_ALERT)) { return false; } return disableAlert; }
From source file:org.apache.oozie.service.ConfigurationService.java
License:Apache License
public static long getLong(Configuration conf, String name) { return conf.getLong(name, ConfigUtils.LONG_DEFAULT); }
From source file:org.apache.parquet.hadoop.ParquetInputFormat.java
License:Apache License
/** * @param configuration the configuration to connect to the file system * @param footers the footers of the files to read * @return the splits for the footers/*www .j a va 2 s . com*/ * @throws IOException * @deprecated split planning using file footers will be removed */ @Deprecated public List<ParquetInputSplit> getSplits(Configuration configuration, List<Footer> footers) throws IOException { boolean strictTypeChecking = configuration.getBoolean(STRICT_TYPE_CHECKING, true); final long maxSplitSize = configuration.getLong("mapred.max.split.size", Long.MAX_VALUE); final long minSplitSize = Math.max(getFormatMinSplitSize(), configuration.getLong("mapred.min.split.size", 0L)); if (maxSplitSize < 0 || minSplitSize < 0) { throw new ParquetDecodingException( "maxSplitSize or minSplitSize should not be negative: maxSplitSize = " + maxSplitSize + "; minSplitSize = " + minSplitSize); } GlobalMetaData globalMetaData = ParquetFileWriter.getGlobalMetaData(footers, strictTypeChecking); ReadContext readContext = getReadSupport(configuration).init( new InitContext(configuration, globalMetaData.getKeyValueMetaData(), globalMetaData.getSchema())); return new ClientSideMetadataSplitStrategy().getSplits(configuration, footers, maxSplitSize, minSplitSize, readContext); }