List of usage examples for org.apache.hadoop.conf Configuration iterator
@Override
public Iterator<Map.Entry<String, String>> iterator()
String key-value pairs in the configuration. From source file:cascading.avro.TrevniSchemeTest.java
License:Apache License
@Test public void testSpecifiedColumns() throws Exception { final Schema schema = new Schema.Parser() .parse(getClass().getResourceAsStream("electric-power-usage.avsc")); final Schema specifiedColumnsSchema = new Schema.Parser() .parse(getClass().getResourceAsStream("electric-power-usage2.avsc")); Configuration hadoopConf = new Configuration(); // compression codec for trevni column block. // KKr - This fails on systems without Snappy installed, so commenting it out // hadoopConf.set("trevni.meta.trevni.codec", "snappy"); Map<Object, Object> confMap = new HashMap<Object, Object>(); Iterator<Entry<String, String>> iter = hadoopConf.iterator(); while (iter.hasNext()) { Entry<String, String> entry = iter.next(); confMap.put(entry.getKey(), entry.getValue()); }/*from w w w.j a v a2 s . com*/ JobConf jobConf = new JobConf(hadoopConf); String in = tempDir.getRoot().toString() + "/specifiedColumns/in"; String out = tempDir.getRoot().toString() + "/specifiedColumns/out"; final Fields fields = new Fields("addressCode", "timestamp", "devicePowerEventList"); final Fields innerFields = new Fields("power", "deviceType", "deviceId", "status"); Tap lfsSource = new Lfs(new TrevniScheme(schema), in, SinkMode.REPLACE); TupleEntryCollector write = lfsSource.openForWrite(new HadoopFlowProcess(jobConf)); List<TupleEntry> devicePowerEventList = new ArrayList<TupleEntry>(); devicePowerEventList.add(new TupleEntry(innerFields, new Tuple(1300.0, 5, 0, 1))); devicePowerEventList.add(new TupleEntry(innerFields, new Tuple(3500.4, 4, 1, 0))); List<TupleEntry> devicePowerEventList2 = new ArrayList<TupleEntry>(); devicePowerEventList2.add(new TupleEntry(innerFields, new Tuple(3570.0, 3, 0, 1))); devicePowerEventList2.add(new TupleEntry(innerFields, new Tuple(110.4, 2, 1, 0))); devicePowerEventList2.add(new TupleEntry(innerFields, new Tuple(250.9, 3, 3, 1))); write.add(new TupleEntry(fields, new Tuple("4874025000-514", 1356998460000L, devicePowerEventList))); write.add(new TupleEntry(fields, new Tuple("4725033000-4031", 1356998520000L, devicePowerEventList2))); write.close(); Pipe writePipe = new Pipe("tuples to trevni"); Tap lfsTrevniSource = new Lfs(new TrevniScheme(schema), in + "/*"); Tap trevniSink = new Lfs(new TrevniScheme(schema), out); Flow flow = new HadoopFlowConnector(confMap).connect(lfsTrevniSource, trevniSink, writePipe); flow.complete(); // Read the specified columns. Tap trevniSource = new Lfs(new TrevniScheme(specifiedColumnsSchema), out + "/*"); TupleEntryIterator iterator = trevniSource.openForRead(new HadoopFlowProcess(jobConf)); assertTrue(iterator.hasNext()); final TupleEntry readEntry1 = iterator.next(); assertTrue(readEntry1.getString("addressCode").equals("4874025000-514")); assertEquals(2, ((List) readEntry1.getObject("devicePowerEventList")).size()); assertEquals(1300.0, ((Tuple) ((List) readEntry1.getObject("devicePowerEventList")).get(0)).getDouble(0)); final TupleEntry readEntry2 = iterator.next(); assertTrue(readEntry2.getString("addressCode").equals("4725033000-4031")); assertEquals(3, ((List) readEntry2.getObject("devicePowerEventList")).size()); assertEquals(110.4, ((Tuple) ((List) readEntry1.getObject("devicePowerEventList")).get(1)).getDouble(0)); }
From source file:com.bah.culvert.configuration.ConfigurationTest.java
License:Apache License
@Test public void incorporatesNewConfItems() { Configuration conf = new Configuration(); String v1 = conf.iterator().next().getKey(); conf.set(v1, "a special value"); Configuration c2 = CConfiguration.create(conf); Assert.assertEquals("a special value", c2.get(v1)); }
From source file:com.blackberry.logdriver.admin.LogMaintenance.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = getConf(); // If run by Oozie, then load the Oozie conf too if (System.getProperty("oozie.action.conf.xml") != null) { conf.addResource(new URL("file://" + System.getProperty("oozie.action.conf.xml"))); }/*w w w. j av a 2 s . c o m*/ // For some reason, Oozie needs some options to be set in system instead of // in the confiuration. So copy the configs over. { Iterator<Entry<String, String>> i = conf.iterator(); while (i.hasNext()) { Entry<String, String> next = i.next(); System.setProperty(next.getKey(), next.getValue()); } } if (args.length < 3) { printUsage(); return 1; } String userName = args[0]; String dcNumber = args[1]; String service = args[2]; String date = null; String hour = null; if (args.length >= 4) { date = args[3]; } if (args.length >= 5) { hour = args[4]; } // Set from environment variables String mergeJobPropertiesFile = getConfOrEnv(conf, "MERGEJOB_CONF"); String filterJobPropertiesFile = getConfOrEnv(conf, "FILTERJOB_CONF"); String daysBeforeArchive = getConfOrEnv(conf, "DAYS_BEFORE_ARCHIVE"); String daysBeforeDelete = getConfOrEnv(conf, "DAYS_BEFORE_DELETE"); String maxConcurrentMR = getConfOrEnv(conf, "MAX_CONCURRENT_MR", "-1"); String zkConnectString = getConfOrEnv(conf, "ZK_CONNECT_STRING"); String logdir = getConfOrEnv(conf, "logdriver.logdir.name"); boolean resetOrphanedJobs = Boolean.parseBoolean(getConfOrEnv(conf, "reset.orphaned.jobs", "true")); String rootDir = getConfOrEnv(conf, "service.root.dir"); String maxTotalMR = getConfOrEnv(conf, "MAX_TOTAL_MR", "-1"); boolean doMerge = true; boolean doArchive = true; boolean doDelete = true; if (zkConnectString == null) { LOG.error("ZK_CONNECT_STRING is not set. Exiting."); return 1; } if (mergeJobPropertiesFile == null) { LOG.info("MERGEJOB_CONF is not set. Not merging."); doMerge = false; } if (filterJobPropertiesFile == null) { LOG.info("FILTERJOB_CONF is not set. Not archiving."); doArchive = false; } if (daysBeforeArchive == null) { LOG.info("DAYS_BEFORE_ARCHIVE is not set. Not archiving."); doArchive = false; } if (doArchive && Integer.parseInt(daysBeforeArchive) < 0) { LOG.info("DAYS_BEFORE_ARCHIVE is negative. Not archiving."); doArchive = false; } if (daysBeforeDelete == null) { LOG.info("DAYS_BEFORE_DELETE is not set. Not deleting."); doDelete = false; } if (doDelete && Integer.parseInt(daysBeforeDelete) < 0) { LOG.info("DAYS_BEFORE_DELETE is negative. Not deleting."); doDelete = false; } if (logdir == null) { LOG.info("LOGDRIVER_LOGDIR_NAME is not set. Using default value of 'logs'."); logdir = "logs"; } if (rootDir == null) { LOG.info("SERVICE_ROOT_DIR is not set. Using default value of 'service'."); rootDir = "/service"; } // We can hang if this fails. So make sure we abort if it fails. fs = null; try { fs = FileSystem.get(conf); fs.exists(new Path("/")); // Test if it works. } catch (IOException e) { LOG.error("Error getting filesystem.", e); return 1; } // Create the LockUtil instance lockUtil = new LockUtil(zkConnectString); // Now it's safe to create our Job Runner JobRunner jobRunner = new JobRunner(Integer.parseInt(maxConcurrentMR), Integer.parseInt(maxTotalMR)); Thread jobRunnerThread = new Thread(jobRunner); jobRunnerThread.setName("JobRunner"); jobRunnerThread.setDaemon(false); jobRunnerThread.start(); // Figure out what date we start filters on. String filterCutoffDate = ""; if (doArchive) { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, Integer.parseInt("-" + daysBeforeArchive)); filterCutoffDate = String.format("%04d%02d%02d%02d", cal.get(Calendar.YEAR), (cal.get(Calendar.MONTH) + 1), cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY)); LOG.info("Archiving logs from before {}", filterCutoffDate); } String deleteCutoffDate = ""; if (doDelete) { Calendar cal = Calendar.getInstance(); cal.add(Calendar.DAY_OF_MONTH, Integer.parseInt("-" + daysBeforeDelete)); deleteCutoffDate = String.format("%04d%02d%02d%02d", cal.get(Calendar.YEAR), (cal.get(Calendar.MONTH) + 1), cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY)); LOG.info("Deleting logs from before {}", deleteCutoffDate); } long now = System.currentTimeMillis(); // Various exceptions have been popping up here. So make sure I catch them // all. try { // Patterns to recognize hour, day and incoming directories, so that they // can be processed. Pattern datePathPattern; Pattern hourPathPattern; Pattern incomingPathPattern; Pattern dataPathPattern; Pattern archivePathPattern; Pattern workingPathPattern; if (hour != null) { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")"); hourPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")"); incomingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/incoming"); dataPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/data"); archivePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/archive"); workingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(" + Pattern.quote(hour) + ")/([^/]+)/working/([^/]+)_(\\d+)"); } else if (date != null) { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")"); hourPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})"); incomingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/incoming"); dataPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/data"); archivePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/archive"); workingPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(" + Pattern.quote(date) + ")/(\\d{2})/([^/]+)/working/([^/]+)_(\\d+)"); } else { datePathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})"); hourPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})"); incomingPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/incoming"); dataPathPattern = Pattern.compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/data"); archivePathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/archive"); workingPathPattern = Pattern .compile(rootDir + "/" + Pattern.quote(dcNumber) + "/" + Pattern.quote(service) + "/" + Pattern.quote(logdir) + "/(\\d{8})/(\\d{2})/([^/]+)/working/([^/]+)_(\\d+)"); } // Do a depth first search of the directory, processing anything that // looks // interesting along the way Deque<Path> paths = new ArrayDeque<Path>(); Path rootPath = new Path(rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/"); paths.push(rootPath); while (paths.size() > 0) { Path p = paths.pop(); LOG.debug("{}", p.toString()); if (!fs.exists(p)) { continue; } FileStatus dirStatus = fs.getFileStatus(p); FileStatus[] children = fs.listStatus(p); boolean addChildren = true; boolean old = dirStatus.getModificationTime() < now - WAIT_TIME; LOG.debug(" Was last modified {}ms ago", now - dirStatus.getModificationTime()); if (!old) { LOG.debug(" Skipping, since it's not old enough."); } else if ((!rootPath.equals(p)) && (children.length == 0 || (children.length == 1 && children[0].getPath().getName().equals(READY_MARKER)))) { // old and no children? Delete! LOG.info(" Deleting empty directory {}", p.toString()); fs.delete(p, true); } else { Matcher matcher = datePathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking date directory"); // If this is already done, then skip it. So only process if it // doesn't exist. if (fs.exists(new Path(p, READY_MARKER)) == false) { // Check each subdirectory. If they all have ready markers, then I // guess we're ready. boolean ready = true; for (FileStatus c : children) { if (c.isDirectory() && fs.exists(new Path(c.getPath(), READY_MARKER)) == false) { ready = false; break; } } if (ready) { fs.createNewFile(new Path(p, READY_MARKER)); } } } matcher = hourPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking hour directory"); // If this is already done, then skip it. So only process if it // doesn't exist. if (fs.exists(new Path(p, READY_MARKER)) == false) { // Check each subdirectory. If they all have ready markers, then I // guess we're ready. boolean ready = true; for (FileStatus c : children) { if (c.isDirectory() && fs.exists(new Path(c.getPath(), READY_MARKER)) == false) { ready = false; break; } } if (ready) { fs.createNewFile(new Path(p, READY_MARKER)); } } } // Check to see if we have to run a merge matcher = incomingPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.debug("Checking incoming directory"); String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } else if (doMerge) { // old, looks right, and has children? Run it! boolean hasMatchingChildren = false; boolean subdirTooYoung = false; for (FileStatus child : children) { if (!hasMatchingChildren) { FileStatus[] grandchildren = fs.listStatus(child.getPath()); for (FileStatus gc : grandchildren) { if (VALID_FILE.matcher(gc.getPath().getName()).matches()) { hasMatchingChildren = true; break; } } } if (!subdirTooYoung) { if (child.getModificationTime() >= now - WAIT_TIME) { subdirTooYoung = true; LOG.debug(" Subdir {} is too young.", child.getPath()); } } } if (!hasMatchingChildren) { LOG.debug(" No files match the expected pattern ({})", VALID_FILE.pattern()); } if (hasMatchingChildren && !subdirTooYoung) { LOG.info(" Run Merge job {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); Properties jobProps = new Properties(); jobProps.load(new FileInputStream(mergeJobPropertiesFile)); jobProps.setProperty("jobType", "merge"); jobProps.setProperty("rootDir", rootDir); jobProps.setProperty("dcNumber", dcNumber); jobProps.setProperty("service", service); jobProps.setProperty("date", matchDate); jobProps.setProperty("hour", matchHour); jobProps.setProperty("component", matchComponent); jobProps.setProperty("user.name", userName); jobProps.setProperty("logdir", logdir); jobRunner.submit(jobProps); addChildren = false; } } } // Check to see if we need to run a filter and archive matcher = dataPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } else if (doArchive && timestamp.compareTo(filterCutoffDate) < 0) { Properties jobProps = new Properties(); jobProps.load(new FileInputStream(filterJobPropertiesFile)); jobProps.setProperty("jobType", "filter"); jobProps.setProperty("rootDir", rootDir); jobProps.setProperty("dcNumber", dcNumber); jobProps.setProperty("service", service); jobProps.setProperty("date", matchDate); jobProps.setProperty("hour", matchHour); jobProps.setProperty("component", matchComponent); jobProps.setProperty("user.name", userName); jobProps.setProperty("logdir", logdir); // Check to see if we should just keep all or delete all here. // The filter file should be here String appPath = jobProps.getProperty("oozie.wf.application.path"); appPath = appPath.replaceFirst("\\$\\{.*?\\}", ""); Path filterFile = new Path( appPath + "/" + conf.get("filter.definition.file", service + ".yaml")); LOG.info("Filter file is {}", filterFile); if (fs.exists(filterFile)) { List<BoomFilterMapper.Filter> filters = BoomFilterMapper.loadFilters(matchComponent, fs.open(filterFile)); if (filters == null) { LOG.warn( " Got null when getting filters. Not processing. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); } else if (filters.size() == 0) { LOG.warn(" Got no filters. Not processing. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); } else if (filters.size() == 1 && filters.get(0) instanceof BoomFilterMapper.KeepAllFilter) { LOG.info(" Keeping everything. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); // Move files from data to archive // delete it all! String destination = rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/" + matchDate + "/" + matchHour + "/" + matchComponent + "/archive/"; PathInfo pathInfo = new PathInfo(); pathInfo.setDcNumber(dcNumber); pathInfo.setService(service); pathInfo.setLogdir(logdir); pathInfo.setDate(matchDate); pathInfo.setHour(matchHour); pathInfo.setComponent(matchComponent); try { lockUtil.acquireWriteLock(lockUtil.getLockPath(pathInfo)); fs.mkdirs(new Path(destination)); for (FileStatus f : fs.listStatus(p)) { fs.rename(f.getPath(), new Path(destination)); } } finally { lockUtil.releaseWriteLock(lockUtil.getLockPath(pathInfo)); } } else if (filters.size() == 1 && filters.get(0) instanceof BoomFilterMapper.DropAllFilter) { LOG.info(" Dropping everything. {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); PathInfo pathInfo = new PathInfo(); pathInfo.setDcNumber(dcNumber); pathInfo.setService(service); pathInfo.setLogdir(logdir); pathInfo.setDate(matchDate); pathInfo.setHour(matchHour); pathInfo.setComponent(matchComponent); try { lockUtil.acquireWriteLock(lockUtil.getLockPath(pathInfo)); fs.delete(p, true); } finally { lockUtil.releaseWriteLock(lockUtil.getLockPath(pathInfo)); } } else { LOG.info(" Run Filter/Archive job {} :: {} {} {} {} {}", new Object[] { p.toString(), dcNumber, service, matchDate, matchHour, matchComponent }); jobRunner.submit(jobProps); } } else { LOG.warn("Skipping filter job, since no filter file exists"); } addChildren = false; } } matcher = archivePathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String timestamp = matchDate + matchHour; if (doDelete && timestamp.compareTo(deleteCutoffDate) < 0) { LOG.info("Deleting old directory: {}", p); fs.delete(p, true); addChildren = false; } } matcher = workingPathPattern.matcher(p.toUri().getPath()); if (matcher.matches()) { LOG.info(" Matches working pattern ({})", p); if (resetOrphanedJobs) { String matchDate = matcher.group(1); String matchHour = matcher.group(2); String matchComponent = matcher.group(3); // Move everything from working/xxx/incoming/ to incoming/ PathInfo lockPathInfo = new PathInfo(logdir, rootDir + "/" + dcNumber + "/" + service + "/" + logdir + "/" + matchDate + "/" + matchHour + "/" + matchComponent); lockUtil.acquireWriteLock(lockUtil.getLockPath(lockPathInfo)); FileStatus[] fileStatuses = fs.listStatus(new Path(p.toUri().getPath() + "/incoming/")); if (fileStatuses != null) { for (FileStatus fileStatus : fileStatuses) { Path toPath = new Path( fileStatus.getPath().getParent().getParent().getParent().getParent(), "incoming/" + fileStatus.getPath().getName()); LOG.info(" Moving data from {} to {}", fileStatus.getPath(), toPath); LOG.info(" mkdir {}", toPath); fs.mkdirs(toPath); Path fromDir = new Path(p.toUri().getPath(), "incoming/" + fileStatus.getPath().getName()); LOG.info(" moving from {}", fromDir); FileStatus[] files = fs.listStatus(fromDir); if (files == null || files.length == 0) { LOG.info(" Nothing to move from {}", fromDir); } else { for (FileStatus f : files) { LOG.info(" rename {} {}", f.getPath(), new Path(toPath, f.getPath().getName())); fs.rename(f.getPath(), new Path(toPath, f.getPath().getName())); } } LOG.info(" rm {}", fileStatus.getPath()); fs.delete(fileStatus.getPath(), true); } lockUtil.releaseWriteLock(lockUtil.getLockPath(lockPathInfo)); fs.delete(new Path(p.toUri().getPath()), true); } } addChildren = false; } } // Add any children which are directories to the stack. if (addChildren) { for (int i = children.length - 1; i >= 0; i--) { FileStatus child = children[i]; if (child.isDirectory()) { paths.push(child.getPath()); } } } } // Since we may have deleted a bunch of directories, delete any unused // locks // from ZooKeeper. { LOG.info("Checking for unused locks in ZooKeeper"); String scanPath = rootDir + "/" + dcNumber + "/" + service + "/" + logdir; if (date != null) { scanPath += "/" + date; if (hour != null) { scanPath += "/" + hour; } } List<LockInfo> lockInfo = lockUtil.scan(scanPath); for (LockInfo li : lockInfo) { // Check if the lock path still exists in HDFS. If it doesn't, then // delete it from ZooKeeper. String path = li.getPath(); String hdfsPath = path.substring(LockUtil.ROOT.length()); if (!fs.exists(new Path(hdfsPath))) { ZooKeeper zk = lockUtil.getZkClient(); while (!path.equals(LockUtil.ROOT)) { try { zk.delete(path, -1); } catch (KeeperException.NotEmptyException e) { // That's fine. just stop trying then. break; } catch (Exception e) { LOG.error("Caught exception trying to delete from ZooKeeper.", e); break; } LOG.info("Deleted from ZooKeeper: {}", path); path = path.substring(0, path.lastIndexOf('/')); } } } } // Now that we're done, wait for the Oozie Runner to stop, and print the // results. LOG.info("Waiting for Oozie jobs to complete."); jobRunner.shutdown(); jobRunnerThread.join(); LOG.info("Job Stats : Started={} Succeeded={} failed={} errors={}", new Object[] { jobRunner.getStarted(), jobRunner.getSucceeded(), jobRunner.getFailed(), jobRunner.getErrors() }); lockUtil.close(); } catch (Exception e) { LOG.error("Unexpected exception caught.", e); return 1; } return 0; }
From source file:com.datatorrent.stram.cli.ApexCli.java
License:Apache License
DTConfiguration getLaunchAppPackageProperties(AppPackage ap, ConfigPackage cp,
LaunchCommandLineInfo commandLineInfo, String appName) throws Exception {
DTConfiguration launchProperties = new DTConfiguration();
List<AppInfo> applications = getAppsFromPackageAndConfig(ap, cp, commandLineInfo.useConfigApps);
AppInfo selectedApp = null;/* w w w. j ava 2s. c o m*/
for (AppInfo app : applications) {
if (app.name.equals(appName)) {
selectedApp = app;
break;
}
}
Map<String, String> defaultProperties = selectedApp == null ? ap.getDefaultProperties()
: selectedApp.defaultProperties;
Set<String> requiredProperties = new TreeSet<>(
selectedApp == null ? ap.getRequiredProperties() : selectedApp.requiredProperties);
for (Map.Entry<String, String> entry : defaultProperties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
// settings specified in the user's environment take precedence over defaults in package.
// since both are merged into a single -conf option below, apply them on top of the defaults here.
File confFile = new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE);
if (confFile.exists()) {
Configuration userConf = new Configuration(false);
userConf.addResource(new Path(confFile.toURI()));
Iterator<Entry<String, String>> it = userConf.iterator();
while (it.hasNext()) {
Entry<String, String> entry = it.next();
// filter relevant entries
if (entry.getKey().startsWith(StreamingApplication.DT_PREFIX)) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
}
if (commandLineInfo.apConfigFile != null) {
DTConfiguration givenConfig = new DTConfiguration();
givenConfig.loadFile(new File(ap.tempDirectory() + "/conf/" + commandLineInfo.apConfigFile));
for (Map.Entry<String, String> entry : givenConfig) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
if (cp != null) {
Map<String, String> properties = cp.getProperties(appName);
for (Map.Entry<String, String> entry : properties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
} else if (commandLineInfo.configFile != null) {
DTConfiguration givenConfig = new DTConfiguration();
givenConfig.loadFile(new File(commandLineInfo.configFile));
for (Map.Entry<String, String> entry : givenConfig) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
if (commandLineInfo.overrideProperties != null) {
for (Map.Entry<String, String> entry : commandLineInfo.overrideProperties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
// now look at whether it is in default configuration
for (Map.Entry<String, String> entry : conf) {
if (StringUtils.isNotBlank(entry.getValue())) {
requiredProperties.remove(entry.getKey());
}
}
if (!requiredProperties.isEmpty()) {
throw new CliException("Required properties not set: " + StringUtils.join(requiredProperties, ", "));
}
//StramClientUtils.evalProperties(launchProperties);
return launchProperties;
}
From source file:com.datatorrent.stram.cli.DTCli.java
License:Apache License
DTConfiguration getLaunchAppPackageProperties(AppPackage ap, ConfigPackage cp,
LaunchCommandLineInfo commandLineInfo, String appName) throws Exception {
DTConfiguration launchProperties = new DTConfiguration();
List<AppInfo> applications = ap.getApplications();
AppInfo selectedApp = null;// w w w.jav a 2s .c om
for (AppInfo app : applications) {
if (app.name.equals(appName)) {
selectedApp = app;
break;
}
}
Map<String, String> defaultProperties = selectedApp == null ? ap.getDefaultProperties()
: selectedApp.defaultProperties;
Set<String> requiredProperties = new TreeSet<String>(
selectedApp == null ? ap.getRequiredProperties() : selectedApp.requiredProperties);
for (Map.Entry<String, String> entry : defaultProperties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
// settings specified in the user's environment take precedence over defaults in package.
// since both are merged into a single -conf option below, apply them on top of the defaults here.
File confFile = new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE);
if (confFile.exists()) {
Configuration userConf = new Configuration(false);
userConf.addResource(new Path(confFile.toURI()));
Iterator<Entry<String, String>> it = userConf.iterator();
while (it.hasNext()) {
Entry<String, String> entry = it.next();
// filter relevant entries
if (entry.getKey().startsWith(StreamingApplication.DT_PREFIX)) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
}
if (commandLineInfo.apConfigFile != null) {
DTConfiguration givenConfig = new DTConfiguration();
givenConfig.loadFile(new File(ap.tempDirectory() + "/conf/" + commandLineInfo.apConfigFile));
for (Map.Entry<String, String> entry : givenConfig) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
if (cp != null) {
Map<String, String> properties = cp.getProperties(appName);
for (Map.Entry<String, String> entry : properties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
} else if (commandLineInfo.configFile != null) {
DTConfiguration givenConfig = new DTConfiguration();
givenConfig.loadFile(new File(commandLineInfo.configFile));
for (Map.Entry<String, String> entry : givenConfig) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
if (commandLineInfo.overrideProperties != null) {
for (Map.Entry<String, String> entry : commandLineInfo.overrideProperties.entrySet()) {
launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
requiredProperties.remove(entry.getKey());
}
}
// now look at whether it is in default configuration
for (Map.Entry<String, String> entry : conf) {
if (StringUtils.isNotBlank(entry.getValue())) {
requiredProperties.remove(entry.getKey());
}
}
if (!requiredProperties.isEmpty()) {
throw new CliException("Required properties not set: " + StringUtils.join(requiredProperties, ", "));
}
//StramClientUtils.evalProperties(launchProperties);
return launchProperties;
}
From source file:com.datatorrent.stram.client.StramClientUtils.java
License:Apache License
@SuppressWarnings("deprecation") private static void convertDeprecatedProperties(Configuration conf) { Iterator<Map.Entry<String, String>> iterator = conf.iterator(); Map<String, String> newEntries = new HashMap<String, String>(); while (iterator.hasNext()) { Map.Entry<String, String> entry = iterator.next(); if (entry.getKey().startsWith("stram.")) { String newKey = StreamingApplication.DT_PREFIX + entry.getKey().substring(6); LOG.warn("Configuration property {} is deprecated. Please use {} instead.", entry.getKey(), newKey); newEntries.put(newKey, entry.getValue()); iterator.remove();//from ww w .j a va 2s . c o m } } for (Map.Entry<String, String> entry : newEntries.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } }
From source file:com.datatorrent.stram.plan.logical.LogicalPlanConfiguration.java
License:Apache License
public static Properties toProperties(Configuration conf, String prefix) { Iterator<Entry<String, String>> it = conf.iterator(); Properties props = new Properties(); while (it.hasNext()) { Entry<String, String> e = it.next(); // filter relevant entries if (e.getKey().startsWith(prefix)) { props.put(e.getKey(), e.getValue()); }/*w ww. j av a 2 s.c om*/ } return props; }
From source file:com.ebay.erl.mobius.core.MobiusJob.java
License:Apache License
/** * Return the Hadoop job configuration.//w ww. j ava 2s. com * <p> * Note that, this method creates a new {@link Configuration} * from the default one every time, so changes that are made * to the returned {@link Configuration} won't affect the conf * returned by the next call of {@link #getConf()}. */ @Override public Configuration getConf() { Configuration conf = super.getConf() == null ? new Configuration() : super.getConf(); Configuration clone = new Configuration(); Iterator<Entry<String, String>> it = conf.iterator(); while (it.hasNext()) { Entry<String, String> entry = it.next(); clone.set(entry.getKey(), entry.getValue()); } return clone; }
From source file:com.ebay.erl.mobius.util.Util.java
License:Apache License
/** * Merge the given <code>confs</code> into ones. * <p>// w w w . j a va 2 s . c o m * * The value from same property key in the later * configuration objects in the <code>confs</code> * will override the previous one. * * @return a new Configuration that has all the values * in the given <code>confs</code> list. */ public static Configuration merge(Configuration... confs) { Configuration newConf = new Configuration(false); for (Configuration aConf : confs) { Iterator<Entry<String, String>> it = aConf.iterator(); while (it.hasNext()) { Entry<String, String> anEntry = it.next(); if (anEntry.getKey().equals(ConfigureConstants.DATASET_ID_TO_NAME_MAPPING)) { // handle ConfigureConstants.DATASET_ID_TO_NAME_MAPPING differently, as // this key is set by per dataset, Configuration generated by each dataset // is independent with each other. String existingMapping = newConf.get(ConfigureConstants.DATASET_ID_TO_NAME_MAPPING, ""); if (existingMapping.isEmpty()) { newConf.set(ConfigureConstants.DATASET_ID_TO_NAME_MAPPING, anEntry.getValue()); } else { newConf.set(ConfigureConstants.DATASET_ID_TO_NAME_MAPPING, existingMapping + "," + anEntry.getValue()); } } else { newConf.set(anEntry.getKey(), anEntry.getValue()); } } } return newConf; }
From source file:com.flipkart.aesop.runtime.producer.hbase.HBaseEventProducer.java
License:Apache License
/** * Interface method implementation. Starts up the SEP consumer * @see com.linkedin.databus2.producers.EventProducer#start(long) *//*from ww w .j a va 2s. co m*/ public void start(long sinceSCN) { shutdownRequested.set(false); this.sinceSCN.set(sinceSCN); LOGGER.info("Starting SEP subscription : " + this.getName()); LOGGER.info("ZK quorum hosts : " + this.zkQuorum); LOGGER.info("ZK client port : " + this.zkClientPort); LOGGER.info("ZK session timeout : " + this.zkSessionTimeout); LOGGER.info("RPC timeout : " + this.rpcTimeout); LOGGER.info("Using hostname to bind to : " + this.localHost); LOGGER.info("Using worker threads : " + this.workerThreads); LOGGER.info("Listening to WAL edits from : " + this.sinceSCN); try { Configuration hbaseConf = HBaseConfiguration.create(); // enable replication to get WAL edits hbaseConf.setBoolean(HBASE_REPLICATION_CONFIG, true); // need to explicitly set the ZK host and port details - hosts separated from port - see SepModelImpl constructor source code hbaseConf.set(ZK_QUORUM_CONFIG, this.zkQuorum); hbaseConf.setInt(ZK_CLIENT_PORT_CONFIG, this.zkClientPort); hbaseConf.setInt(RPC_TIMEOUT_CONFIG, this.rpcTimeout); StringBuilder zkQuorumWithPort = new StringBuilder(); String[] zkHostsList = this.zkQuorum.split(","); for (String zkHost : zkHostsList) { zkQuorumWithPort.append(zkHost); zkQuorumWithPort.append(":"); zkQuorumWithPort.append(this.zkClientPort); zkQuorumWithPort.append(","); } LOGGER.info("ZK util connect string (host:port) : " + zkQuorumWithPort.toString()); ZooKeeperItf zk = ZkUtil.connect(zkQuorumWithPort.toString(), this.zkSessionTimeout); StringBuilder hbaseConfBuilder = new StringBuilder(); Iterator<Entry<String, String>> it = hbaseConf.iterator(); while (it.hasNext()) { Entry<String, String> entry = it.next(); if (entry.getKey().equalsIgnoreCase(HBASE_REPLICATION_CONFIG) || entry.getKey().equalsIgnoreCase(ZK_QUORUM_CONFIG) || entry.getKey().equalsIgnoreCase(ZK_CLIENT_PORT_CONFIG)) { hbaseConfBuilder.append(entry.getKey()); hbaseConfBuilder.append(":"); hbaseConfBuilder.append(entry.getValue()); hbaseConfBuilder.append(","); } } LOGGER.info("SEP Model Hbase configuration = " + hbaseConfBuilder.toString()); SepModel sepModel = new SepModelImpl(zk, hbaseConf); final String subscriptionName = this.getName(); if (!sepModel.hasSubscription(subscriptionName)) { sepModel.addSubscriptionSilent(subscriptionName); } this.sepConsumer = new SepConsumer(subscriptionName, generateSEPSCN(this.sinceSCN.get()), new RelayAppender(), this.workerThreads, this.localHost, zk, hbaseConf); this.sepConsumer.start(); } catch (Exception e) { LOGGER.error( "Error starting WAL edits consumer. Producer not started!. Error message : " + e.getMessage(), e); } }