Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.falcon.regression.core.util; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.helpers.entity.AbstractEntityHelper; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.client.AuthOozieClient; import org.apache.oozie.client.BundleJob; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.OozieClientException; import org.apache.oozie.client.Job; import org.apache.oozie.client.CoordinatorAction; import org.apache.oozie.client.CoordinatorJob; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; import org.joda.time.DateTime; import org.apache.log4j.Logger; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.json.JSONException; import org.testng.Assert; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; /** * helper methods for oozie . */ public final class OozieUtil { public static final String FAIL_MSG = "NO_such_workflow_exists"; private OozieUtil() { throw new AssertionError("Instantiating utility class..."); } private static final Logger LOGGER = Logger.getLogger(OozieUtil.class); public static AuthOozieClient getClient(String url) { return new AuthOozieClient(url); } public static List<BundleJob> getBundles(OozieClient client, String filter, int start, int len) throws OozieClientException { LOGGER.info("Connecting to oozie: " + client.getOozieUrl()); return client.getBundleJobsInfo(filter, start, len); } public static List<String> getBundleIds(List<BundleJob> bundles) { List<String> ids = new ArrayList<>(); for (BundleJob bundle : bundles) { LOGGER.info("Bundle Id: " + bundle.getId()); ids.add(bundle.getId()); } return ids; } public static List<Job.Status> getBundleStatuses(List<BundleJob> bundles) { List<Job.Status> statuses = new ArrayList<>(); for (BundleJob bundle : bundles) { LOGGER.info("bundle: " + bundle); statuses.add(bundle.getStatus()); } return statuses; } public static String getMaxId(List<String> ids) { String oozieId = ids.get(0); int maxInt = Integer.valueOf(oozieId.split("-")[0]); for (int i = 1; i < ids.size(); i++) { String currentId = ids.get(i); int currInt = Integer.valueOf(currentId.split("-")[0]); if (currInt > maxInt) { oozieId = currentId; } } return oozieId; } public static String getMinId(List<String> ids) { String oozieId = ids.get(0); int minInt = Integer.valueOf(oozieId.split("-")[0]); for (int i = 1; i < ids.size(); i++) { String currentId = ids.get(i); int currInt = Integer.valueOf(currentId.split("-")[0]); if (currInt < minInt) { oozieId = currentId; } } return oozieId; } /** * @param bundleID bundle number * @param oozieClient oozie client * @return list of action ids of the succeeded retention workflow * @throws OozieClientException */ public static List<String> waitForRetentionWorkflowToSucceed(String bundleID, OozieClient oozieClient) throws OozieClientException { LOGGER.info("Connecting to oozie: " + oozieClient.getOozieUrl()); List<String> jobIds = new ArrayList<>(); LOGGER.info("using bundleId:" + bundleID); waitForCoordinatorJobCreation(oozieClient, bundleID); final String coordinatorId = oozieClient.getBundleJobInfo(bundleID).getCoordinators().get(0).getId(); LOGGER.info("using coordinatorId: " + coordinatorId); for (int i = 0; i < 120 && oozieClient.getCoordJobInfo(coordinatorId).getActions().isEmpty(); ++i) { TimeUtil.sleepSeconds(4); } Assert.assertFalse(oozieClient.getCoordJobInfo(coordinatorId).getActions().isEmpty(), "Coordinator actions should have got created by now."); CoordinatorAction action = oozieClient.getCoordJobInfo(coordinatorId).getActions().get(0); for (int i = 0; i < 180; ++i) { CoordinatorAction actionInfo = oozieClient.getCoordActionInfo(action.getId()); LOGGER.info("actionInfo: " + actionInfo); if (EnumSet.of(CoordinatorAction.Status.SUCCEEDED, CoordinatorAction.Status.KILLED, CoordinatorAction.Status.FAILED).contains(actionInfo.getStatus())) { break; } TimeUtil.sleepSeconds(10); } Assert.assertEquals(oozieClient.getCoordActionInfo(action.getId()).getStatus(), CoordinatorAction.Status.SUCCEEDED, "Action did not succeed."); jobIds.add(action.getId()); return jobIds; } public static void waitForCoordinatorJobCreation(OozieClient oozieClient, String bundleID) throws OozieClientException { LOGGER.info("Connecting to oozie: " + oozieClient.getOozieUrl()); for (int i = 0; i < 60 && oozieClient.getBundleJobInfo(bundleID).getCoordinators().isEmpty(); ++i) { TimeUtil.sleepSeconds(2); } Assert.assertFalse(oozieClient.getBundleJobInfo(bundleID).getCoordinators().isEmpty(), "Coordinator job should have got created by now."); } public static Job.Status getOozieJobStatus(OozieClient client, String processName, EntityType entityType) throws OozieClientException { String filter = String.format("name=FALCON_%s_%s", entityType, processName); List<Job.Status> statuses = getBundleStatuses(getBundles(client, filter, 0, 10)); if (statuses.isEmpty()) { return null; } else { return statuses.get(0); } } public static List<String> getBundles(OozieClient client, String entityName, EntityType entityType) throws OozieClientException { String filter = "name=FALCON_" + entityType + "_" + entityName; return getBundleIds(getBundles(client, filter, 0, 10)); } public static List<DateTime> getStartTimeForRunningCoordinators(ColoHelper prismHelper, String bundleID) throws OozieClientException { List<DateTime> startTimes = new ArrayList<>(); OozieClient oozieClient = prismHelper.getClusterHelper().getOozieClient(); BundleJob bundleJob = oozieClient.getBundleJobInfo(bundleID); CoordinatorJob jobInfo; for (CoordinatorJob job : bundleJob.getCoordinators()) { if (job.getAppName().contains("DEFAULT")) { jobInfo = oozieClient.getCoordJobInfo(job.getId()); for (CoordinatorAction action : jobInfo.getActions()) { DateTime temp = new DateTime(action.getCreatedTime(), DateTimeZone.UTC); LOGGER.info(temp); startTimes.add(temp); } } Collections.sort(startTimes); if (!(startTimes.isEmpty())) { return startTimes; } } return null; } public static boolean verifyOozieJobStatus(OozieClient client, String processName, EntityType entityType, Job.Status expectedStatus) throws OozieClientException { for (int seconds = 0; seconds < 100; seconds += 5) { Job.Status status = getOozieJobStatus(client, processName, entityType); LOGGER.info("Current status: " + status); if (status == expectedStatus) { return true; } TimeUtil.sleepSeconds(5); } return false; } public static List<String> getMissingDependencies(OozieClient oozieClient, String bundleID) throws OozieClientException { CoordinatorJob jobInfo; jobInfo = null; BundleJob bundleJob = oozieClient.getBundleJobInfo(bundleID); List<CoordinatorJob> coordinatorJobList = bundleJob.getCoordinators(); if (coordinatorJobList.size() > 1) { for (CoordinatorJob coord : bundleJob.getCoordinators()) { LOGGER.info("Appname is : " + coord.getAppName()); if ((coord.getAppName().contains("DEFAULT") && coord.getAppName().contains("PROCESS")) || (coord.getAppName().contains("REPLICATION") && coord.getAppName().contains("FEED"))) { jobInfo = oozieClient.getCoordJobInfo(coord.getId()); } else { LOGGER.info("Desired coord does not exists on " + oozieClient.getOozieUrl()); } } } else { jobInfo = oozieClient.getCoordJobInfo(bundleJob.getCoordinators().get(0).getId()); } LOGGER.info("Coordinator id : " + jobInfo); List<CoordinatorAction> actions = null; if (jobInfo != null) { actions = jobInfo.getActions(); } if (actions != null) { if (actions.size() < 1) { return null; } } if (actions != null) { LOGGER.info("conf from event: " + actions.get(0).getMissingDependencies()); } String[] missingDependencies = new String[0]; if (actions != null) { missingDependencies = actions.get(0).getMissingDependencies().split("#"); } return new ArrayList<>(Arrays.asList(missingDependencies)); } public static List<String> getWorkflowJobs(OozieClient oozieClient, String bundleID) throws OozieClientException { waitForCoordinatorJobCreation(oozieClient, bundleID); List<String> workflowIds = new ArrayList<>(); List<CoordinatorJob> coordJobs = oozieClient.getBundleJobInfo(bundleID).getCoordinators(); CoordinatorJob coordJobInfo = oozieClient.getCoordJobInfo(coordJobs.get(0).getId()); for (CoordinatorAction action : coordJobInfo.getActions()) { workflowIds.add(action.getExternalId()); } return workflowIds; } public static List<String> getWorkflow(OozieClient oozieClient, String bundleID) throws OozieClientException { waitForCoordinatorJobCreation(oozieClient, bundleID); List<String> workflowIds = new ArrayList<>(); String coordId = getDefaultCoordIDFromBundle(oozieClient, bundleID); CoordinatorJob coordJobInfo = oozieClient.getCoordJobInfo(coordId); for (CoordinatorAction action : coordJobInfo.getActions()) { if (action.getStatus().name().equals("RUNNING") || action.getStatus().name().equals("SUCCEEDED")) { workflowIds.add(action.getExternalId()); } if (action.getStatus().name().equals("KILLED") || action.getStatus().name().equals("WAITING")) { Assert.assertNull(action.getExternalId()); } } return workflowIds; } public static Date getNominalTime(OozieClient oozieClient, String bundleID) throws OozieClientException { BundleJob bundleJob = oozieClient.getBundleJobInfo(bundleID); CoordinatorJob jobInfo = oozieClient.getCoordJobInfo(bundleJob.getCoordinators().get(0).getId()); List<CoordinatorAction> actions = jobInfo.getActions(); return actions.get(0).getNominalTime(); } public static CoordinatorJob getDefaultOozieCoord(OozieClient oozieClient, String bundleId, EntityType type) throws OozieClientException { BundleJob bundlejob = oozieClient.getBundleJobInfo(bundleId); for (CoordinatorJob coord : bundlejob.getCoordinators()) { if ((coord.getAppName().contains("DEFAULT") && EntityType.PROCESS == type) || (coord.getAppName().contains("REPLICATION") && EntityType.FEED == type)) { return oozieClient.getCoordJobInfo(coord.getId()); } else { LOGGER.info("Desired coord does not exists on " + oozieClient.getOozieUrl()); } } return null; } public static int getNumberOfWorkflowInstances(OozieClient oozieClient, String bundleId) throws OozieClientException { return getDefaultOozieCoord(oozieClient, bundleId, EntityType.PROCESS).getActions().size(); } public static List<String> getActionsNominalTime(OozieClient oozieClient, String bundleId, EntityType type) throws OozieClientException { Map<Date, CoordinatorAction.Status> actions = getActionsNominalTimeAndStatus(oozieClient, bundleId, type); List<String> nominalTime = new ArrayList<>(); for (Date date : actions.keySet()) { nominalTime.add(date.toString()); } return nominalTime; } public static Map<Date, CoordinatorAction.Status> getActionsNominalTimeAndStatus(OozieClient oozieClient, String bundleId, EntityType type) throws OozieClientException { Map<Date, CoordinatorAction.Status> result = new TreeMap<>(); List<CoordinatorAction> actions = getDefaultOozieCoord(oozieClient, bundleId, type).getActions(); for (CoordinatorAction action : actions) { result.put(action.getNominalTime(), action.getStatus()); } return result; } public static boolean isBundleOver(ColoHelper coloHelper, String bundleId) throws OozieClientException { OozieClient client = coloHelper.getClusterHelper().getOozieClient(); BundleJob bundleJob = client.getBundleJobInfo(bundleId); if (EnumSet.of(BundleJob.Status.DONEWITHERROR, BundleJob.Status.FAILED, BundleJob.Status.SUCCEEDED, BundleJob.Status.KILLED).contains(bundleJob.getStatus())) { return true; } TimeUtil.sleepSeconds(20); return false; } public static void verifyNewBundleCreation(OozieClient oozieClient, String originalBundleId, List<String> initialNominalTimes, String entity, boolean shouldBeCreated, boolean matchInstances) throws OozieClientException { String entityName = Util.readEntityName(entity); EntityType entityType = Util.getEntityType(entity); String newBundleId = getLatestBundleID(oozieClient, entityName, entityType); if (shouldBeCreated) { Assert.assertTrue(!newBundleId.equalsIgnoreCase(originalBundleId), "eeks! new bundle is not getting created!!!!"); LOGGER.info("old bundleId=" + originalBundleId + " on oozie: " + oozieClient); LOGGER.info("new bundleId=" + newBundleId + " on oozie: " + oozieClient); if (matchInstances) { validateNumberOfWorkflowInstances(oozieClient, initialNominalTimes, originalBundleId, newBundleId, entityType); } } else { Assert.assertEquals(newBundleId, originalBundleId, "eeks! new bundle is getting created!!!!"); } } private static void validateNumberOfWorkflowInstances(OozieClient oozieClient, List<String> initialNominalTimes, String originalBundleId, String newBundleId, EntityType type) throws OozieClientException { List<String> nominalTimesOriginalAndNew = getActionsNominalTime(oozieClient, originalBundleId, type); nominalTimesOriginalAndNew.addAll(getActionsNominalTime(oozieClient, newBundleId, type)); initialNominalTimes.removeAll(nominalTimesOriginalAndNew); if (initialNominalTimes.size() != 0) { LOGGER.info("Missing instance are : " + initialNominalTimes); LOGGER.debug("Original Bundle ID : " + originalBundleId); LOGGER.debug("New Bundle ID : " + newBundleId); Assert.fail("some instances have gone missing after update"); } } public static String getCoordStartTime(OozieClient oozieClient, String entity, int bundleNo) throws OozieClientException { String bundleID = getSequenceBundleID(oozieClient, Util.readEntityName(entity), Util.getEntityType(entity), bundleNo); CoordinatorJob coord = getDefaultOozieCoord(oozieClient, bundleID, Util.getEntityType(entity)); return TimeUtil.dateToOozieDate(coord.getStartTime()); } public static DateTimeFormatter getOozieDateTimeFormatter() { return DateTimeFormat.forPattern("yyyy'-'MM'-'dd'T'HH':'mm'Z'"); } public static int getNumberOfBundle(OozieClient oozieClient, EntityType type, String entityName) throws OozieClientException { return OozieUtil.getBundles(oozieClient, entityName, type).size(); } public static void createMissingDependencies(ColoHelper helper, EntityType type, String entityName, int bundleNumber, int instanceNumber) throws OozieClientException, IOException { final OozieClient oozieClient = helper.getClusterHelper().getOozieClient(); String bundleID = getSequenceBundleID(oozieClient, entityName, type, bundleNumber); List<CoordinatorJob> coords = oozieClient.getBundleJobInfo(bundleID).getCoordinators(); final List<String> missingDependencies = getMissingDependenciesForInstance(oozieClient, coords, instanceNumber); HadoopUtil.createFolders(helper.getClusterHelper().getHadoopFS(), helper.getPrefix(), missingDependencies); } private static List<String> getMissingDependenciesForInstance(OozieClient oozieClient, List<CoordinatorJob> coords, int instanceNumber) throws OozieClientException { ArrayList<String> missingPaths = new ArrayList<>(); for (CoordinatorJob coord : coords) { CoordinatorJob temp = oozieClient.getCoordJobInfo(coord.getId()); CoordinatorAction instance = temp.getActions().get(instanceNumber); missingPaths.addAll(Arrays.asList(instance.getMissingDependencies().split("#"))); } return missingPaths; } public static List<List<String>> createMissingDependencies(ColoHelper helper, EntityType type, String entityName, int bundleNumber) throws OozieClientException, IOException { final OozieClient oozieClient = helper.getClusterHelper().getOozieClient(); String bundleID = getSequenceBundleID(oozieClient, entityName, type, bundleNumber); return createMissingDependenciesForBundle(helper, bundleID); } public static List<List<String>> createMissingDependenciesForBundle(ColoHelper helper, String bundleId) throws OozieClientException, IOException { OozieClient oozieClient = helper.getClusterHelper().getOozieClient(); List<CoordinatorJob> coords = oozieClient.getBundleJobInfo(bundleId).getCoordinators(); List<List<String>> missingDependencies = getMissingDependenciesForBundle(oozieClient, coords); for (List<String> missingDependencyPerInstance : missingDependencies) { HadoopUtil.createFolders(helper.getClusterHelper().getHadoopFS(), helper.getPrefix(), missingDependencyPerInstance); } return missingDependencies; } private static List<List<String>> getMissingDependenciesForBundle(OozieClient oozieClient, List<CoordinatorJob> coords) throws OozieClientException, IOException { List<List<String>> missingDependencies = new ArrayList<>(); for (CoordinatorJob coord : coords) { CoordinatorJob temp = oozieClient.getCoordJobInfo(coord.getId()); for (int instanceNumber = 0; instanceNumber < temp.getActions().size(); instanceNumber++) { CoordinatorAction instance = temp.getActions().get(instanceNumber); missingDependencies.add(Arrays.asList(instance.getMissingDependencies().split("#"))); } } return missingDependencies; } public static void validateRetryAttempts(OozieClient oozieClient, String bundleId, EntityType type, int attempts) throws OozieClientException { CoordinatorJob coord = getDefaultOozieCoord(oozieClient, bundleId, type); int actualRun = oozieClient.getJobInfo(coord.getActions().get(0).getExternalId()).getRun(); LOGGER.info("Actual run count: " + actualRun); // wrt 0 Assert.assertEquals(actualRun, attempts, "Rerun attempts did not match"); } /** * Try to find feed coordinators of given type. */ public static int checkIfFeedCoordExist(OozieClient oozieClient, String feedName, String coordType) throws OozieClientException { return checkIfFeedCoordExist(oozieClient, feedName, coordType, 5); } /** * Try to find feed coordinators of given type given number of times. */ public static int checkIfFeedCoordExist(OozieClient oozieClient, String feedName, String coordType, int numberOfRetries) throws OozieClientException { LOGGER.info("feedName: " + feedName); for (int retryAttempt = 0; retryAttempt < numberOfRetries; retryAttempt++) { int numberOfCoord = 0; List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED); if (bundleIds.size() == 0) { TimeUtil.sleepSeconds(4); continue; } LOGGER.info("bundleIds: " + bundleIds); for (String aBundleId : bundleIds) { LOGGER.info("aBundleId: " + aBundleId); waitForCoordinatorJobCreation(oozieClient, aBundleId); List<CoordinatorJob> coords = getBundleCoordinators(oozieClient, aBundleId); LOGGER.info("coords: " + coords); for (CoordinatorJob coord : coords) { if (coord.getAppName().contains(coordType)) { numberOfCoord++; } } } if (numberOfCoord > 0) { return numberOfCoord; } TimeUtil.sleepSeconds(4); } return 0; } /** * Retrieves replication coordinatorID from bundle of coordinators. */ public static List<String> getReplicationCoordID(String bundleId, AbstractEntityHelper helper) throws OozieClientException { final OozieClient oozieClient = helper.getOozieClient(); List<CoordinatorJob> coords = getBundleCoordinators(oozieClient, bundleId); List<String> replicationCoordID = new ArrayList<>(); for (CoordinatorJob coord : coords) { if (coord.getAppName().contains("FEED_REPLICATION")) { replicationCoordID.add(coord.getId()); } } return replicationCoordID; } /** * Retrieves ID of bundle related to some process/feed using its ordinal number. * * @param entityName - name of entity bundle is related to * @param entityType - feed or process * @param bundleNumber - ordinal number of bundle * @return bundle ID * @throws org.apache.oozie.client.OozieClientException */ public static String getSequenceBundleID(OozieClient oozieClient, String entityName, EntityType entityType, int bundleNumber) throws OozieClientException { //sequence start from 0 List<String> bundleIds = getBundles(oozieClient, entityName, entityType); Collections.sort(bundleIds); if (bundleNumber < bundleIds.size()) { return bundleIds.get(bundleNumber); } return null; } /** * Retrieves the latest bundle ID. * * @param oozieClient where job is running * @param entityName name of entity job is related to * @param entityType type of entity - feed or process expected * @return latest bundle ID * @throws org.apache.oozie.client.OozieClientException */ public static String getLatestBundleID(OozieClient oozieClient, String entityName, EntityType entityType) throws OozieClientException { List<String> bundleIds = getBundles(oozieClient, entityName, entityType); String max = "0"; int maxID = -1; for (String strID : bundleIds) { if (maxID < Integer.parseInt(strID.substring(0, strID.indexOf('-')))) { maxID = Integer.parseInt(strID.substring(0, strID.indexOf('-'))); max = strID; } } return max; } /** * Retrieves all coordinators of bundle. * * @param oozieClient Oozie client to use for fetching info. * @param bundleID specific bundle ID * @return list of bundle coordinators * @throws org.apache.oozie.client.OozieClientException */ public static List<CoordinatorJob> getBundleCoordinators(OozieClient oozieClient, String bundleID) throws OozieClientException { BundleJob bundleInfo = oozieClient.getBundleJobInfo(bundleID); return bundleInfo.getCoordinators(); } public static Job.Status getDefaultCoordinatorStatus(OozieClient oozieClient, String processName, int bundleNumber) throws OozieClientException { String bundleID = getSequenceBundleID(oozieClient, processName, EntityType.PROCESS, bundleNumber); String coordId = getDefaultCoordIDFromBundle(oozieClient, bundleID); return oozieClient.getCoordJobInfo(coordId).getStatus(); } public static String getDefaultCoordIDFromBundle(OozieClient oozieClient, String bundleId) throws OozieClientException { waitForCoordinatorJobCreation(oozieClient, bundleId); BundleJob bundleInfo = oozieClient.getBundleJobInfo(bundleId); List<CoordinatorJob> coords = bundleInfo.getCoordinators(); int min = 100000; String minString = ""; for (CoordinatorJob coord : coords) { String strID = coord.getId(); if (min > Integer.parseInt(strID.substring(0, strID.indexOf('-')))) { min = Integer.parseInt(strID.substring(0, strID.indexOf('-'))); minString = coord.getId(); } } LOGGER.info("function getDefaultCoordIDFromBundle: minString: " + minString); return minString; } public static String getLatestCoordinatorID(OozieClient oozieClient, String processName, EntityType entityType) throws OozieClientException { final String latestBundleID = getLatestBundleID(oozieClient, processName, entityType); return getDefaultCoordIDFromBundle(oozieClient, latestBundleID); } /** * Waits till bundle job will reach expected status. * Generates time according to expected status. * * @param oozieClient oozieClient of cluster job is running on * @param processName name of process which job is being analyzed * @param expectedStatus job status we are waiting for * @throws org.apache.oozie.client.OozieClientException */ public static void waitForBundleToReachState(OozieClient oozieClient, String processName, Job.Status expectedStatus) throws OozieClientException { int totalMinutesToWait = getMinutesToWait(expectedStatus); waitForBundleToReachState(oozieClient, processName, expectedStatus, totalMinutesToWait); } /** * Waits till bundle job will reach expected status during specific time. * Use it directly in test cases when timeouts are different from trivial, in other cases use * waitForBundleToReachState(OozieClient, String, Status) * * @param oozieClient oozie client of cluster job is running on * @param processName name of process which job is being analyzed * @param expectedStatus job status we are waiting for * @param totalMinutesToWait specific time to wait expected state * @throws org.apache.oozie.client.OozieClientException */ public static void waitForBundleToReachState(OozieClient oozieClient, String processName, Job.Status expectedStatus, int totalMinutesToWait) throws OozieClientException { int sleep = totalMinutesToWait * 60 / 20; for (int sleepCount = 0; sleepCount < sleep; sleepCount++) { String bundleID = getLatestBundleID(oozieClient, processName, EntityType.PROCESS); BundleJob j = oozieClient.getBundleJobInfo(bundleID); LOGGER.info(sleepCount + ". Current status: " + j.getStatus() + "; expected: " + expectedStatus); if (j.getStatus() == expectedStatus) { return; } TimeUtil.sleepSeconds(20); } Assert.fail("State " + expectedStatus + " wasn't reached in " + totalMinutesToWait + " mins"); } /** * Generates time which is presumably needed for bundle job to reach particular state. * * @param expectedStatus status which we are expect to get from bundle job * @return minutes to wait for expected status */ private static int getMinutesToWait(Job.Status expectedStatus) { switch (expectedStatus) { case DONEWITHERROR: case SUCCEEDED: return OSUtil.IS_WINDOWS ? 40 : 20; case KILLED: return OSUtil.IS_WINDOWS ? 30 : 15; default: return OSUtil.IS_WINDOWS ? 60 : 30; } } public static String getActionStatus(OozieClient oozieClient, String workflowId, String actionName) throws OozieClientException { List<WorkflowAction> wfAction = oozieClient.getJobInfo(workflowId).getActions(); for (WorkflowAction wf : wfAction) { if (wf.getName().contains(actionName)) { return wf.getExternalStatus(); } } return ""; } public static String getWorkflowActionStatus(OozieClient oozieClient, String bundleId, String actionName) throws OozieClientException { List<String> workflowIds = getWorkflowJobs(oozieClient, bundleId); if (workflowIds.get(0).isEmpty()) { return FAIL_MSG; } return getActionStatus(oozieClient, workflowIds.get(0), actionName); } public static String getSubWorkflowActionStatus(OozieClient oozieClient, String bundleId, String actionName, String subAction) throws OozieClientException { List<String> workflowIds = getWorkflowJobs(oozieClient, bundleId); if (workflowIds.get(0).isEmpty()) { return FAIL_MSG; } String wid = ""; List<WorkflowAction> wfAction = oozieClient.getJobInfo(workflowIds.get(0)).getActions(); for (WorkflowAction wf : wfAction) { if (wf.getName().contains(actionName)) { wid = wf.getExternalId(); } } if (!wid.isEmpty()) { return getActionStatus(oozieClient, wid, subAction); } return FAIL_MSG; } /** * Returns configuration object of a given bundleID for a given instanceTime. * * @param oozieClient oozie client of cluster job is running on * @param bundleID bundleID of given cluster * @param time instanceTime * @throws org.apache.oozie.client.OozieClientException * @throws org.json.JSONException */ public static Configuration getProcessConf(OozieClient oozieClient, String bundleID, String time) throws OozieClientException, JSONException { waitForCoordinatorJobCreation(oozieClient, bundleID); List<CoordinatorJob> coordJobs = oozieClient.getBundleJobInfo(bundleID).getCoordinators(); CoordinatorJob coordJobInfo = oozieClient.getCoordJobInfo(coordJobs.get(0).getId()); Configuration conf = new Configuration(); for (CoordinatorAction action : coordJobInfo.getActions()) { String dateStr = (new DateTime(action.getNominalTime(), DateTimeZone.UTC)).toString(); if (!dateStr.isEmpty() && dateStr.contains(time.replace("Z", ""))) { conf.addResource(new ByteArrayInputStream( oozieClient.getJobInfo(action.getExternalId()).getConf().getBytes())); } } return conf; } /** * Method retrieves and parses replication coordinator action workflow definition and checks whether specific * properties are present in list of workflow args or not. * @param workflowDefinition workflow definition * @param actionName action within workflow, e.g pre-processing, replication etc. * @param propMap specific properties which are expected to be in arg list * @return true if all keys and values are present, false otherwise */ public static boolean propsArePresentInWorkflow(String workflowDefinition, String actionName, HashMap<String, String> propMap) { //get action definition Document definition = Util.convertStringToDocument(workflowDefinition); Assert.assertNotNull(definition, "Workflow definition shouldn't be null."); NodeList actions = definition.getElementsByTagName("action"); Element action = null; for (int i = 0; i < actions.getLength(); i++) { Node node = actions.item(i); if (node.getNodeType() == Node.ELEMENT_NODE) { action = (Element) node; if (action.getAttribute("name").equals(actionName)) { break; } action = null; } } Assert.assertNotNull(action, actionName + " action not found."); //retrieve and checks whether properties are present in workflow args Element javaElement = (Element) action.getElementsByTagName("java").item(0); NodeList args = javaElement.getElementsByTagName("arg"); int counter = 0; String key = null; for (int i = 0; i < args.getLength(); i++) { Node node = args.item(i); if (node.getNodeType() == Node.ELEMENT_NODE) { String argKey = node.getTextContent().replace("-", ""); if (key != null && propMap.get(key).equals(argKey)) { counter++; key = null; } else if (key == null && propMap.containsKey(argKey)) { key = argKey; } } } return counter == propMap.size(); } /** * Returns configuration object of a given bundleID for a given retentionFeed. * * @param oozieClient oozie client of cluster job is running on * @param bundleID bundleID of given cluster * @throws OozieClientException */ public static Configuration getRetentionConfiguration(OozieClient oozieClient, String bundleID) throws OozieClientException { waitForCoordinatorJobCreation(oozieClient, bundleID); CoordinatorJob coord = null; List<CoordinatorJob> coordJobs = oozieClient.getBundleJobInfo(bundleID).getCoordinators(); for (CoordinatorJob coordinatorJob : coordJobs) { if (coordinatorJob.getAppName().startsWith("FALCON_FEED_RETENTION")) { coord = oozieClient.getCoordJobInfo(coordinatorJob.getId()); } } Configuration configuration = new Configuration(); if (coord != null) { WorkflowJob wid = oozieClient.getJobInfo(coord.getActions().get(0).getExternalId()); configuration.addResource(new ByteArrayInputStream(wid.getConf().getBytes())); } else { configuration = null; } return configuration; } }