org.apache.eagle.jpm.mr.history.parser.JHFEventReaderBase.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.eagle.jpm.mr.history.parser.JHFEventReaderBase.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

package org.apache.eagle.jpm.mr.history.parser;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.eagle.jpm.mr.history.MRHistoryJobConfig;
import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter;
import org.apache.eagle.jpm.mr.history.metrics.JobCounterMetricsGenerator;
import org.apache.eagle.jpm.mr.historyentity.*;
import org.apache.eagle.jpm.util.Constants;
import org.apache.eagle.jpm.util.JobNameNormalization;
import org.apache.eagle.jpm.util.MRJobTagName;
import org.apache.eagle.jpm.util.Utils;
import org.apache.eagle.jpm.util.jobcounter.JobCounters;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.jobhistory.EventType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Closeable;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * A new instance of JHFEventReaderBase will be created for each job history log file.
 */
public abstract class JHFEventReaderBase extends JobEntityCreationPublisher implements Closeable {
    private static final Logger LOG = LoggerFactory.getLogger(JHFEventReaderBase.class);
    protected Map<String, String> baseTags;
    protected JobEventAPIEntity jobSubmitEventEntity;
    protected JobEventAPIEntity jobLaunchEventEntity;
    protected int numTotalMaps;
    protected int numTotalReduces;
    protected JobEventAPIEntity jobFinishEventEntity;
    protected JobExecutionAPIEntity jobExecutionEntity;
    protected Map<String, Long> taskStartTime;
    // taskAttemptID to task attempt startTime
    protected Map<String, Long> taskAttemptStartTime;

    // taskID to host mapping, for task it's the host where the last attempt runs on
    protected Map<String, String> taskRunningHosts;
    // hostname to rack mapping
    protected Map<String, String> host2RackMapping;
    // taskattempt to error msg, attemptId, taskId, error
    protected Map<String, Pair<String, String>> attempt2ErrorMsg;

    protected String jobId;
    protected String jobName;
    protected String jobType;
    protected String jobDefId;
    protected String user;
    protected String queueName;
    protected Long jobLaunchTime;
    protected JobHistoryContentFilter filter;

    protected final List<HistoryJobEntityLifecycleListener> jobEntityLifecycleListeners = new ArrayList<>();

    protected final Configuration configuration;

    private long sumMapTaskDuration;
    private long sumReduceTaskDuration;

    private JobCounterMetricsGenerator jobCounterMetricsGenerator;

    private MRHistoryJobConfig appConfig;

    /**
     * baseTags stores the basic tag name values which might be used for persisting various entities.
     * baseTags includes: site and jobName
     * baseTags are used for all job/task related entities
     *
     * @param baseTags
     */
    public JHFEventReaderBase(Map<String, String> baseTags, Configuration configuration,
            JobHistoryContentFilter filter, MRHistoryJobConfig appConfig) {
        this.filter = filter;

        this.baseTags = baseTags;
        jobSubmitEventEntity = new JobEventAPIEntity();
        jobSubmitEventEntity.setTags(new HashMap<>(baseTags));

        jobLaunchEventEntity = new JobEventAPIEntity();
        jobLaunchEventEntity.setTags(new HashMap<>(baseTags));

        jobFinishEventEntity = new JobEventAPIEntity();
        jobFinishEventEntity.setTags(new HashMap<>(baseTags));

        jobExecutionEntity = new JobExecutionAPIEntity();
        jobExecutionEntity.setTags(new HashMap<>(baseTags));
        jobExecutionEntity.setNumFailedMaps(0);
        jobExecutionEntity.setNumFailedReduces(0);

        taskRunningHosts = new HashMap<>();

        host2RackMapping = new HashMap<>();
        attempt2ErrorMsg = new HashMap<>();

        taskStartTime = new HashMap<>();
        taskAttemptStartTime = new HashMap<>();

        this.configuration = configuration;

        if (this.configuration != null && this.jobType == null) {
            this.setJobType(Utils.fetchJobType(this.configuration).toString());
        }
        this.sumMapTaskDuration = 0L;
        this.sumReduceTaskDuration = 0L;

        this.appConfig = appConfig;
        this.jobCounterMetricsGenerator = new JobCounterMetricsGenerator(appConfig.getEagleServiceConfig());
    }

    public void register(HistoryJobEntityLifecycleListener lifecycleListener) {
        this.jobEntityLifecycleListeners.add(lifecycleListener);
    }

    @Override
    public void close() throws IOException {
        // check if this job history file is complete
        if (jobExecutionEntity.getEndTime() == 0L) {
            throw new IOException(new JHFWriteNotCompletedException(jobId));
        }
        try {
            flush();
            this.jobCounterMetricsGenerator.flush();
        } catch (Exception ex) {
            throw new IOException(ex);
        }
    }

    @Override
    public void flush() throws Exception {
        super.flush();
        for (HistoryJobEntityLifecycleListener listener : this.jobEntityLifecycleListeners) {
            listener.flush();
        }
    }

    private String buildJobTrackingUrl(String jobId) {
        String jobTrackingUrlBase = appConfig.getJobHistoryEndpointConfig().mrHistoryServerUrl + "/jobhistory/job/";
        try {
            URI oldUri = new URI(jobTrackingUrlBase);
            URI resolved = oldUri.resolve(jobId);
            return resolved.toString();
        } catch (URISyntaxException e) {
            LOG.warn("Tracking url build failed with baseURL=%s, resolvePart=%s", jobTrackingUrlBase, jobId);
            return jobTrackingUrlBase;
        }
    }

    /**
     * ...
     * @param id
     */
    private void setJobID(String id) {
        this.jobId = id;
    }

    private void setJobType(String jobType) {
        this.jobType = jobType;
    }

    protected void handleJob(EventType eventType, Map<Keys, String> values, Object totalCounters) throws Exception {
        String id = values.get(Keys.JOBID);

        if (jobId == null) {
            setJobID(id);
        } else if (!jobId.equals(id)) {
            String msg = "Current job ID '" + id + "' does not match previously stored value '" + jobId + "'";
            LOG.error(msg);
            throw new ImportException(msg);
        }

        if (values.get(Keys.SUBMIT_TIME) != null) { // job submitted
            jobSubmitEventEntity.setTimestamp(Long.valueOf(values.get(Keys.SUBMIT_TIME)));
            user = values.get(Keys.USER);
            queueName = values.get(Keys.JOB_QUEUE);
            jobName = values.get(Keys.JOBNAME);

            // If given job name then use it as norm job name, otherwise use eagle JobNameNormalization rule to generate.
            String jobDefId = null;
            if (configuration != null) {
                jobDefId = configuration.get(filter.getJobNameKey());
            }

            if (jobDefId == null) {
                this.jobDefId = JobNameNormalization.getInstance(this.appConfig.getConfig()).normalize(jobName);
            } else {
                LOG.debug("Got JobDefId from job configuration for " + id + ": " + jobDefId);
                this.jobDefId = jobDefId;
            }

            LOG.info("JobDefId of " + id + ": " + this.jobDefId);

            jobSubmitEventEntity.getTags().put(MRJobTagName.USER.toString(), user);
            jobSubmitEventEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId);
            jobSubmitEventEntity.getTags().put(MRJobTagName.JOB_STATUS.toString(), EagleJobStatus.SUBMITTED.name());
            jobSubmitEventEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName);
            jobSubmitEventEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), this.jobDefId);
            jobExecutionEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType);
            entityCreated(jobSubmitEventEntity);
        } else if (values.get(Keys.LAUNCH_TIME) != null) { // job launched
            jobLaunchEventEntity.setTimestamp(Long.valueOf(values.get(Keys.LAUNCH_TIME)));
            jobLaunchTime = jobLaunchEventEntity.getTimestamp();
            jobLaunchEventEntity.getTags().put(MRJobTagName.USER.toString(), user);
            jobLaunchEventEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId);
            jobLaunchEventEntity.getTags().put(MRJobTagName.JOB_STATUS.toString(), EagleJobStatus.LAUNCHED.name());
            jobLaunchEventEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName);
            jobLaunchEventEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
            jobLaunchEventEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType);
            numTotalMaps = Integer.valueOf(values.get(Keys.TOTAL_MAPS));
            numTotalReduces = Integer.valueOf(values.get(Keys.TOTAL_REDUCES));
            entityCreated(jobLaunchEventEntity);
        } else if (values.get(Keys.FINISH_TIME) != null) { // job finished
            jobFinishEventEntity.setTimestamp(Long.valueOf(values.get(Keys.FINISH_TIME)));
            jobFinishEventEntity.getTags().put(MRJobTagName.USER.toString(), user);
            jobFinishEventEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId);
            jobFinishEventEntity.getTags().put(MRJobTagName.JOB_STATUS.toString(), values.get(Keys.JOB_STATUS));
            jobFinishEventEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName);
            jobFinishEventEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
            jobFinishEventEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType);
            entityCreated(jobFinishEventEntity);

            // populate jobExecutionEntity entity
            jobExecutionEntity.getTags().put(MRJobTagName.USER.toString(), user);
            jobExecutionEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId);
            jobExecutionEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName);
            jobExecutionEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
            jobExecutionEntity.getTags().put(MRJobTagName.JOB_QUEUE.toString(), queueName);
            jobExecutionEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), this.jobType);

            jobExecutionEntity.setTrackingUrl(buildJobTrackingUrl(jobId));
            jobExecutionEntity.setCurrentState(values.get(Keys.JOB_STATUS));
            jobExecutionEntity.setStartTime(jobLaunchEventEntity.getTimestamp());
            jobExecutionEntity.setEndTime(jobFinishEventEntity.getTimestamp());
            jobExecutionEntity.setDurationTime(jobExecutionEntity.getEndTime() - jobExecutionEntity.getStartTime());
            jobExecutionEntity.setTimestamp(jobLaunchEventEntity.getTimestamp());
            jobExecutionEntity.setSubmissionTime(jobSubmitEventEntity.getTimestamp());
            if (values.get(Keys.FAILED_MAPS) != null) {
                // for Artemis
                jobExecutionEntity.setNumFailedMaps(Integer.valueOf(values.get(Keys.FAILED_MAPS)));
            }
            if (values.get(Keys.FAILED_REDUCES) != null) {
                // for Artemis
                jobExecutionEntity.setNumFailedReduces(Integer.valueOf(values.get(Keys.FAILED_REDUCES)));
            }
            jobExecutionEntity.setNumFinishedMaps(Integer.valueOf(values.get(Keys.FINISHED_MAPS)));
            jobExecutionEntity.setNumFinishedReduces(Integer.valueOf(values.get(Keys.FINISHED_REDUCES)));
            jobExecutionEntity.setNumTotalMaps(numTotalMaps);
            jobExecutionEntity.setNumTotalReduces(numTotalReduces);
            if (values.get(Keys.COUNTERS) != null || totalCounters != null) {
                JobCounters jobCounters = parseCounters(totalCounters);
                jobExecutionEntity.setJobCounters(jobCounters);
                if (jobCounters.getCounters().containsKey(Constants.JOB_COUNTER)) {
                    Map<String, Long> counters = jobCounters.getCounters().get(Constants.JOB_COUNTER);
                    if (counters.containsKey(Constants.JobCounter.DATA_LOCAL_MAPS.toString())) {
                        jobExecutionEntity.setDataLocalMaps(
                                counters.get(Constants.JobCounter.DATA_LOCAL_MAPS.toString()).intValue());
                    }

                    if (counters.containsKey(Constants.JobCounter.RACK_LOCAL_MAPS.toString())) {
                        jobExecutionEntity.setRackLocalMaps(
                                counters.get(Constants.JobCounter.RACK_LOCAL_MAPS.toString()).intValue());
                    }

                    if (counters.containsKey(Constants.JobCounter.TOTAL_LAUNCHED_MAPS.toString())) {
                        jobExecutionEntity.setTotalLaunchedMaps(
                                counters.get(Constants.JobCounter.TOTAL_LAUNCHED_MAPS.toString()).intValue());
                    }
                }

                if (jobExecutionEntity.getTotalLaunchedMaps() > 0) {
                    jobExecutionEntity.setDataLocalMapsPercentage(jobExecutionEntity.getDataLocalMaps() * 1.0
                            / jobExecutionEntity.getTotalLaunchedMaps());
                    jobExecutionEntity.setRackLocalMapsPercentage(jobExecutionEntity.getRackLocalMaps() * 1.0
                            / jobExecutionEntity.getTotalLaunchedMaps());
                }
            }
            jobExecutionEntity.setAvgMapTaskDuration(this.sumMapTaskDuration * 1.0 / numTotalMaps);
            if (numTotalReduces == 0) {
                jobExecutionEntity.setMaxReduceTaskDuration(0);
                jobExecutionEntity.setAvgReduceTaskDuration(0);
            } else {
                jobExecutionEntity.setAvgReduceTaskDuration(this.sumReduceTaskDuration * 1.0 / numTotalReduces);
            }
            this.jobCounterMetricsGenerator.setBaseTags(jobExecutionEntity.getTags());

            formatDiagnostics(values.get(Keys.DIAGNOSTICS));

            entityCreated(jobExecutionEntity);
        }
    }

    private void formatDiagnostics(String diagnostics) {
        String formatDiagnostics = "";
        if (diagnostics != null) {
            for (String attemptId : attempt2ErrorMsg.keySet()) {
                String taskId = attempt2ErrorMsg.get(attemptId).getLeft();
                String error = attempt2ErrorMsg.get(attemptId).getRight();
                if (diagnostics.contains(taskId)) {
                    formatDiagnostics = error;
                    break;
                }
            }
        }
        jobExecutionEntity.setDiagnostics(formatDiagnostics);
    }

    private void entityCreated(JobBaseAPIEntity entity) throws Exception {
        for (HistoryJobEntityLifecycleListener lifecycleListener : this.jobEntityLifecycleListeners) {
            lifecycleListener.jobEntityCreated(entity);
        }

        // job finished when passing JobExecutionAPIEntity: jobExecutionEntity
        if (entity == this.jobExecutionEntity) {
            for (HistoryJobEntityLifecycleListener lifecycleListener : this.jobEntityLifecycleListeners) {
                lifecycleListener.jobFinish();
            }
        }

        super.notifiyListeners(entity);
    }

    protected abstract JobCounters parseCounters(Object value) throws IOException;

    /**
     * for one task ID, it has several sequential task events, i.e.
     * task_start -> task_attempt_start -> task_attempt_finish -> task_attempt_start -> task_attempt_finish -> ... -> task_end
     *
     * @param values
     * @throws IOException
     */
    @SuppressWarnings("serial")
    protected void handleTask(RecordTypes recType, EventType eventType, final Map<Keys, String> values,
            Object counters) throws Exception {
        String taskAttemptID = values.get(Keys.TASK_ATTEMPT_ID);
        String startTime = values.get(Keys.START_TIME);
        String finishTime = values.get(Keys.FINISH_TIME);
        final String taskType = values.get(Keys.TASK_TYPE);
        final String taskID = values.get(Keys.TASKID);

        Map<String, String> taskBaseTags = new HashMap<String, String>() {
            {
                put(MRJobTagName.TASK_TYPE.toString(), taskType);
                put(MRJobTagName.USER.toString(), user);
                //put(MRJobTagName.JOB_NAME.toString(), _jobName);
                put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
                put(MRJobTagName.JOB_TYPE.toString(), jobType);
                put(MRJobTagName.JOB_ID.toString(), jobId);
                put(MRJobTagName.TASK_ID.toString(), taskID);
            }
        };
        taskBaseTags.putAll(baseTags);
        if (recType == RecordTypes.Task && startTime != null) { // task start, no host is assigned yet
            taskStartTime.put(taskID, Long.valueOf(startTime));
        } else if (recType == RecordTypes.Task && finishTime != null) { // task finish
            // task execution entity setup
            Map<String, String> taskExecutionTags = new HashMap<>(taskBaseTags);
            String hostname = taskRunningHosts.get(taskID);
            hostname = (hostname == null) ? "" : hostname; // TODO if task fails, then no hostname
            taskExecutionTags.put(MRJobTagName.HOSTNAME.toString(), hostname);
            taskExecutionTags.put(MRJobTagName.RACK.toString(), host2RackMapping.get(hostname));
            TaskExecutionAPIEntity entity = new TaskExecutionAPIEntity();
            entity.setTags(taskExecutionTags);
            entity.setStartTime(taskStartTime.get(taskID));
            entity.setEndTime(Long.valueOf(finishTime));
            entity.setDuration(entity.getEndTime() - entity.getStartTime());
            entity.setTimestamp(jobLaunchTime);
            entity.setError(values.get(Keys.ERROR));
            entity.setTaskStatus(values.get(Keys.TASK_STATUS));
            if (values.get(Keys.COUNTERS) != null || counters != null) {
                entity.setJobCounters(parseCounters(counters));
            }
            long duration = entity.getEndTime() - jobLaunchTime;
            if (taskType.equals(Constants.TaskType.MAP.toString())
                    && duration > jobExecutionEntity.getLastMapDuration()) {
                jobExecutionEntity.setLastMapDuration(duration);
            }
            if (taskType.equals(Constants.TaskType.REDUCE.toString())
                    && duration > jobExecutionEntity.getLastReduceDuration()) {
                jobExecutionEntity.setLastReduceDuration(duration);
            }

            if (taskType.equals(Constants.TaskType.MAP.toString())
                    && entity.getDuration() > jobExecutionEntity.getMaxMapTaskDuration()) {
                jobExecutionEntity.setMaxMapTaskDuration(entity.getDuration());
            }
            if (taskType.equals(Constants.TaskType.REDUCE.toString())
                    && entity.getDuration() > jobExecutionEntity.getMaxReduceTaskDuration()) {
                jobExecutionEntity.setMaxReduceTaskDuration(entity.getDuration());
            }

            if (taskType.equals(Constants.TaskType.MAP.toString())) {
                this.sumMapTaskDuration += entity.getDuration();
                if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name())
                        || entity.getTaskStatus().equals(EagleTaskStatus.KILLED.name())) {
                    jobExecutionEntity.setNumFailedMaps(1 + jobExecutionEntity.getNumFailedMaps());
                }
            } else {
                this.sumReduceTaskDuration += entity.getDuration();
                if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name())
                        || entity.getTaskStatus().equals(EagleTaskStatus.KILLED.name())) {
                    jobExecutionEntity.setNumFailedReduces(1 + jobExecutionEntity.getNumFailedReduces());
                }
            }

            entityCreated(entity);
            this.jobCounterMetricsGenerator.taskExecutionEntityCreated(entity);
            //_taskStartTime.remove(taskID); // clean this taskID
        } else if ((recType == RecordTypes.MapAttempt || recType == RecordTypes.ReduceAttempt)
                && startTime != null) { // task attempt start
            taskAttemptStartTime.put(taskAttemptID, Long.valueOf(startTime));
        } else if ((recType == RecordTypes.MapAttempt || recType == RecordTypes.ReduceAttempt)
                && finishTime != null) { // task attempt finish
            TaskAttemptExecutionAPIEntity entity = new TaskAttemptExecutionAPIEntity();
            Map<String, String> taskAttemptExecutionTags = new HashMap<>(taskBaseTags);
            entity.setTags(taskAttemptExecutionTags);
            String hostname = values.get(Keys.HOSTNAME);
            String rack = values.get(Keys.RACK);
            taskAttemptExecutionTags.put(MRJobTagName.HOSTNAME.toString(), hostname);
            taskAttemptExecutionTags.put(MRJobTagName.RACK.toString(), rack);
            taskAttemptExecutionTags.put(MRJobTagName.TASK_ATTEMPT_ID.toString(), taskAttemptID);
            // put last attempt's hostname to task level
            taskRunningHosts.put(taskID, hostname);
            // it is very likely that an attempt ID could be both succeeded and failed due to M/R system
            // in this case, we should ignore this attempt?
            if (taskAttemptStartTime.get(taskAttemptID) == null) {
                LOG.warn("task attemp has consistency issue " + taskAttemptID);
                return;
            }
            entity.setStartTime(taskAttemptStartTime.get(taskAttemptID));
            entity.setEndTime(Long.valueOf(finishTime));
            entity.setTimestamp(jobLaunchTime);
            entity.setDuration(entity.getEndTime() - entity.getStartTime());
            entity.setTaskStatus(values.get(Keys.TASK_STATUS));
            entity.setError(values.get(Keys.ERROR));
            if (values.get(Keys.COUNTERS) != null || counters != null) { // when task is killed, COUNTERS does not exist
                //entity.setJobCounters(parseCounters(values.get(Keys.COUNTERS)));
                entity.setJobCounters(parseCounters(counters));
            }

            if (recType == RecordTypes.MapAttempt) {
                jobExecutionEntity.setTotalMapAttempts(1 + jobExecutionEntity.getTotalMapAttempts());
                if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name())
                        || entity.getTaskStatus().equals(EagleTaskStatus.KILLED.name())) {
                    jobExecutionEntity.setFailedMapAttempts(1 + jobExecutionEntity.getFailedMapAttempts());
                }
            } else {
                jobExecutionEntity.setTotalReduceAttempts(1 + jobExecutionEntity.getTotalReduceAttempts());
                if (entity.getTaskStatus().equals(EagleTaskStatus.FAILED.name())
                        || entity.getTaskStatus().equals(EagleTaskStatus.KILLED.name())) {
                    jobExecutionEntity.setFailedReduceAttempts(1 + jobExecutionEntity.getFailedReduceAttempts());
                }
            }

            entityCreated(entity);
            attempt2ErrorMsg.put(taskAttemptID, Pair.of(taskID, entity.getError()));
            //generate TaskAttemptErrorCategoryEntity
            if (entity.getTags().containsKey(MRJobTagName.ERROR_CATEGORY.toString())) {
                TaskAttemptErrorCategoryEntity taskAttemptErrorCategoryEntity = new TaskAttemptErrorCategoryEntity();
                Map<String, String> taskAttemptErrorCategoryEntityTags = new HashMap<>(entity.getTags());
                taskAttemptErrorCategoryEntity.setTags(taskAttemptErrorCategoryEntityTags);

                taskAttemptErrorCategoryEntity.setStartTime(entity.getStartTime());
                taskAttemptErrorCategoryEntity.setEndTime(entity.getEndTime());
                taskAttemptErrorCategoryEntity.setTimestamp(entity.getTimestamp());
                entityCreated(taskAttemptErrorCategoryEntity);
            }

            taskAttemptStartTime.remove(taskAttemptID);
        } else {
            // silently ignore
            LOG.warn("It's an exceptional case ?");
        }
    }

    public void parseConfiguration() throws Exception {
        Map<String, String> prop = new TreeMap<>();

        if (filter.acceptJobConfFile()) {
            Iterator<Map.Entry<String, String>> iter = configuration.iterator();
            while (iter.hasNext()) {
                String key = iter.next().getKey();
                if (included(key) && !excluded(key)) {
                    prop.put(key, configuration.get(key));
                }
            }
        }

        // check must-have keys are within prop
        if (matchMustHaveKeyPatterns(prop)) {
            JobConfigurationAPIEntity jobConfigurationEntity = new JobConfigurationAPIEntity();
            jobConfigurationEntity.setTags(new HashMap<>(baseTags));
            jobConfigurationEntity.getTags().put(MRJobTagName.USER.toString(), user);
            jobConfigurationEntity.getTags().put(MRJobTagName.JOB_ID.toString(), jobId);
            jobConfigurationEntity.getTags().put(MRJobTagName.JOB_NAME.toString(), jobName);
            jobConfigurationEntity.getTags().put(MRJobTagName.JOD_DEF_ID.toString(), jobDefId);
            jobConfigurationEntity.getTags().put(MRJobTagName.JOB_TYPE.toString(), jobType);
            jobConfigurationEntity.setTimestamp(jobLaunchEventEntity.getTimestamp());

            JobConfig jobConfig = new JobConfig();
            jobConfig.setConfig(prop);
            jobConfigurationEntity.setJobConfig(jobConfig);
            jobConfigurationEntity.setConfigJobName(jobDefId);
            entityCreated(jobConfigurationEntity);
        }
    }

    private boolean matchMustHaveKeyPatterns(Map<String, String> prop) {
        if (filter.getMustHaveJobConfKeyPatterns() == null) {
            return true;
        }

        for (Pattern p : filter.getMustHaveJobConfKeyPatterns()) {
            boolean matched = false;
            for (String key : prop.keySet()) {
                if (p.matcher(key).matches()) {
                    matched = true;
                    break;
                }
            }
            if (!matched) {
                return false;
            }
        }
        return true;
    }

    private boolean included(String key) {
        if (filter.getJobConfKeyInclusionPatterns() == null) {
            return true;
        }
        for (Pattern p : filter.getJobConfKeyInclusionPatterns()) {
            Matcher m = p.matcher(key);
            if (m.matches()) {
                LOG.info("include key: " + p.toString());
                return true;
            }
        }
        return false;
    }

    private boolean excluded(String key) {
        if (filter.getJobConfKeyExclusionPatterns() == null) {
            return false;
        }
        for (Pattern p : filter.getJobConfKeyExclusionPatterns()) {
            Matcher m = p.matcher(key);
            if (m.matches()) {
                return true;
            }
        }
        return false;
    }

    public enum Keys {
        JOBTRACKERID, START_TIME, FINISH_TIME, JOBID, JOBNAME, USER, JOBCONF, SUBMIT_TIME, LAUNCH_TIME, TOTAL_MAPS, TOTAL_REDUCES, FAILED_MAPS, FAILED_REDUCES, FINISHED_MAPS, FINISHED_REDUCES, JOB_STATUS, TASKID, HOSTNAME, TASK_TYPE, ERROR, TASK_ATTEMPT_ID, TASK_STATUS, COPY_PHASE, SORT_PHASE, REDUCE_PHASE, SHUFFLE_FINISHED, SORT_FINISHED, COUNTERS, SPLITS, JOB_PRIORITY, HTTP_PORT, TRACKER_NAME, STATE_STRING, VERSION, MAP_COUNTERS, REDUCE_COUNTERS, VIEW_JOB, MODIFY_JOB, JOB_QUEUE, RACK, DIAGNOSTICS,

        UBERISED, SPLIT_LOCATIONS, FAILED_DUE_TO_ATTEMPT, MAP_FINISH_TIME, PORT, RACK_NAME,

        //For Artemis
        WORKFLOW_ID, WORKFLOW_NAME, WORKFLOW_NODE_NAME, WORKFLOW_ADJACENCIES, WORKFLOW_TAGS, SHUFFLE_PORT, LOCALITY, AVATAAR, FAIL_REASON
    }
}