List of usage examples for org.apache.hadoop.conf Configuration set
public void set(String name, String value)
value of the name property. From source file:com.blackberry.logtools.loggrep.java
License:Apache License
public int run(String[] argv) throws Exception { //Configuring configuration and filesystem to work on HDFS final Configuration conf = getConf(); //Configuration processed by ToolRunner FileSystem fs = FileSystem.get(conf); //Initiate tools used for running search LogTools tools = new LogTools(); //Other options String date_format = "RFC5424"; String field_separator = " "; ArrayList<String> D_options = new ArrayList<String>(); boolean quiet = true; boolean silent = false; boolean log = false; boolean forcelocal = false; boolean forceremote = false; //The arguments are // - regex/*from w ww .j a va 2 s . c o m*/ // - dc number // - service // - component // - startTime (Something 'date' can parse, or just a time in ms from epoch) // - endTime (Same as start) // - outputDir //Indexing for arguments to be passed for Mapreduce int regexNum = 0; int dcNum = 1; int svcNum = 2; int compNum = 3; int startNum = 4; int endNum = 5; int outNum = 6; //Parsing through user arguments String[] args = new String[7]; int count = 0; //Count created to track the parse of all arguments int argcount = 0; //Count created to track number of arguments to be passed on while (count < argv.length) { String arg = argv[count]; count++; if (arg.equals("--")) { break; } else if (arg.startsWith("-")) { if (arg.equals("--v")) { quiet = tools.parseV(silent); } else if (arg.equals("--i")) { conf.set("logdriver.search.case.insensitive", "true"); } else if (arg.startsWith("--dateFormat=")) { arg = arg.replace("--dateFormat=", ""); date_format = arg; } else if (arg.startsWith("--fieldSeparator=")) { arg = arg.replace("--fieldSeparator=", ""); field_separator = arg; } else if (arg.startsWith("-regex=")) { arg = arg.replace("-regex=", ""); args[regexNum] = arg; argcount++; } else if (arg.startsWith("-dc=")) { arg = arg.replace("-dc=", ""); args[dcNum] = arg; argcount++; } else if (arg.startsWith("-svc=")) { arg = arg.replace("-svc=", ""); args[svcNum] = arg; argcount++; } else if (arg.startsWith("-comp=")) { arg = arg.replace("-comp=", ""); args[compNum] = arg; argcount++; } else if (arg.startsWith("-start=")) { arg = arg.replace("-start=", ""); args[startNum] = arg; argcount++; } else if (arg.startsWith("-end=")) { arg = arg.replace("-end=", ""); args[endNum] = arg; argcount++; } //User inputs output directory that is to be created //Check to see if parent directory exists && output directory does not exist else if (arg.startsWith("--out=")) { args[outNum] = tools.parseOut(arg, fs); argcount++; } else if (arg.startsWith("-D")) { D_options.add(arg); } else if (arg.equals("--silent")) { silent = tools.parseSilent(quiet); } else if (arg.equals("--log")) { log = true; } else if (arg.equals("--l")) { forcelocal = tools.parsePigMode(forceremote); } else if (arg.equals("--r")) { forceremote = tools.parsePigMode(forcelocal); } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } //Default output should be stdout represented by "-" if (args[outNum] == null) { args[outNum] = "-"; argcount++; LogTools.logConsole(quiet, silent, info, "Output set to default stdout."); } if (argcount < 7) { System.err.println(";****************************************" + "\n\t\t\t NOT ENOUGH ARGUMENTS\n" + "\n\tUSAGE: loggrep [REQUIRED ARGUMENTS] [OPTIONS] (Order does not matter)" + "\n\tREQUIRED ARGUMENTS:" + "\n\t\t-regex=[REGEX] Java style regular expression." + "\n\t\t-dc=[DATACENTER] Data Center." + "\n\t\t-svc=[SERVICE] Service." + "\n\t\t-comp=[COMPONENT] Component." + "\n\t\t-start=[START] Start time." + "\n\t\t-end=[END] End time." + "\n\tOptions:" + "\n\t\t--out=[DIRECTORY] Desired output directory. If not defined, output to stdout." + "\n\t\t--v Verbose output." + "\n\t\t--r Force remote sort." + "\n\t\t--l Force local sort." + "\n\t\t--dateFormat=[FORMAT] Valid formats are RFC822, RFC3164 (zero padded day)," + "\n\t RFC5424 (default), or any valid format string for FastDateFormat." + "\n\t\t--fieldSeparator=X The separator to use to separate fields in intermediate" + "\n\t files. Defaults to 'INFORMATION SEPARATOR ONE' (U+001F)." + "\n\t\t--silent Output only the data." + "\n\t\t--i Make search case insensitive." + "\n\t\t--log Save all the logs.\n" + ";****************************************"); System.exit(1); } //Parse time inputs for start and end of search args[startNum] = tools.parseDate(args[startNum]); args[endNum] = tools.parseDate(args[endNum]); tools.checkTime(args[startNum], args[endNum]); //Retrieve 'out' argument to determine where output of results should be sent String out = args[outNum]; //Generate files to temporarily store output of mapreduce jobs and pig logs locally File local_output = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { local_output.deleteOnExit(); } File pig_tmp = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { pig_tmp.deleteOnExit(); } //Name the temp directory for storing results in HDFS String tmp = "tmp/loggrep-" + RandomStringUtils.randomAlphanumeric(10); //Set args[outNum] to be temp output directory to be passed onto GrepByTime instead of UserInput argument args[outNum] = (StringEscapeUtils.escapeJava(tmp) + "/rawlines"); //Managing console output - deal with --v/--silent Logger LOG = LoggerFactory.getLogger(loggrep.class); tools.setConsoleOutput(local_output, quiet, silent); //Create temp directory in HDFS to store logsearch logs before sorting tools.tmpDirHDFS(quiet, silent, fs, conf, tmp, log); LogTools.logConsole(quiet, silent, warn, "Searching for " + args[regexNum] + "..."); LogTools.logConsole(quiet, silent, warn, "Passing Arguments: Regex=" + args[regexNum] + " DC=" + args[dcNum] + " Service=" + args[svcNum] + " Component=" + args[compNum] + " StartTime=" + args[startNum] + " EndTime=" + args[endNum] + " Output=" + out); //Set standard configuration for running Mapreduce and PIG String queue_name = "logsearch"; //Start Mapreduce job tools.runMRJob(quiet, silent, conf, D_options, out, LOG, field_separator, queue_name, args, "GrepByTime", new GrepByTime()); //Before sorting, determine the number of records and size of the results found long foundresults = tools.getResults(local_output); long size = tools.getSize(foundresults, tmp, fs); //Run PIG job if results found tools.runPig(silent, quiet, foundresults, size, tmp, out, D_options, queue_name, date_format, field_separator, pig_tmp, fs, conf, forcelocal, forceremote); //Display location of tmp files if log enabled tools.logs(log, local_output, pig_tmp, tmp); return 0; }
From source file:com.blackberry.logtools.logmultisearch.java
License:Apache License
@SuppressWarnings("static-access") public int run(String[] argv) throws Exception { //Configuring configuration and filesystem to work on HDFS final Configuration conf = getConf(); //Configuration processed by ToolRunner FileSystem fs = FileSystem.get(conf); //Initiate tools used for running search LogTools tools = new LogTools(); //Other options String date_format = "RFC5424"; String field_separator = " "; ArrayList<String> D_options = new ArrayList<String>(); boolean quiet = true; boolean silent = false; boolean log = false; boolean forcelocal = false; boolean forceremote = false; //The arguments are // - strings/*from www . ja v a 2s . c o m*/ // - dc number // - service // - component // - startTime (Something 'date' can parse, or just a time in ms from epoch) // - endTime (Same as start) // - outputDir //Indexing for arguments to be passed for Mapreduce int stringsNum = 0; int dcNum = 1; int svcNum = 2; int compNum = 3; int startNum = 4; int endNum = 5; int outNum = 6; //Parsing through user arguments String[] args = new String[7]; int count = 0; //Count created to track the parse of all arguments int argcount = 0; //Count created to track number of arguments to be passed on while (count < argv.length) { String arg = argv[count]; count++; if (arg.equals("--")) { break; } else if (arg.startsWith("-")) { if (arg.equals("--v")) { quiet = tools.parseV(silent); } else if (arg.equals("--i")) { conf.set("logdriver.search.case.insensitive", "true"); } else if (arg.equals("--a")) { LogTools.logConsole(quiet, silent, warn, "AND searching selected"); conf.set("logdriver.search.and", "true"); } else if (arg.startsWith("--dateFormat=")) { arg = arg.replace("--dateFormat=", ""); date_format = arg; } else if (arg.startsWith("--fieldSeparator=")) { arg = arg.replace("--fieldSeparator=", ""); field_separator = arg; } else if (arg.startsWith("-strings=")) { arg = arg.replace("-strings=", ""); args[stringsNum] = arg; argcount++; } else if (arg.startsWith("-dc=")) { arg = arg.replace("-dc=", ""); args[dcNum] = arg; argcount++; } else if (arg.startsWith("-svc=")) { arg = arg.replace("-svc=", ""); args[svcNum] = arg; argcount++; } else if (arg.startsWith("-comp=")) { arg = arg.replace("-comp=", ""); args[compNum] = arg; argcount++; } else if (arg.startsWith("-start=")) { arg = arg.replace("-start=", ""); args[startNum] = arg; argcount++; } else if (arg.startsWith("-end=")) { arg = arg.replace("-end=", ""); args[endNum] = arg; argcount++; } //User inputs output directory that is to be created //Check to see if parent directory exists && output directory does not exist else if (arg.startsWith("--out=")) { args[outNum] = tools.parseOut(arg, fs); argcount++; } else if (arg.startsWith("-D")) { D_options.add(arg); } else if (arg.equals("--silent")) { silent = tools.parseSilent(quiet); } else if (arg.equals("--log")) { log = true; } else if (arg.equals("--l")) { forcelocal = tools.parsePigMode(forceremote); } else if (arg.equals("--r")) { forceremote = tools.parsePigMode(forcelocal); } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } //Default output should be stdout represented by "-" if (args[outNum] == null) { args[outNum] = "-"; argcount++; LogTools.logConsole(quiet, silent, info, "Output set to default stdout."); } if (argcount < 7) { System.err.println(";****************************************" + "\n\t\t\t NOT ENOUGH ARGUMENTS\n" + "\n\tUSAGE: logmultisearch [REQUIRED ARGUMENTS] [OPTIONS] (Order does not matter)" + "\n\tREQUIRED ARGUMENTS:" + "\n\t\t-strings=[STRINGS_DIR|STRINGS_FILE|STRING] String/file/directory of strings to search." + "\n\t\t-dc=[DATACENTER] Data Center." + "\n\t\t-svc=[SERVICE] Service." + "\n\t\t-comp=[COMPONENT] Component." + "\n\t\t-start=[START] Start time." + "\n\t\t-end=[END] End time." + "\n\tOptions:" + "\n\t\t--out=[DIRECTORY] Desired output directory. If not defined, output to stdout." + "\n\t\t--v Verbose output." + "\n\t\t--r Force remote sort." + "\n\t\t--l Force local sort." + "\n\t\t--dateFormat=[FORMAT] Valid formats are RFC822, RFC3164 (zero padded day)," + "\n\t RFC5424 (default), or any valid format string for FastDateFormat." + "\n\t\t--fieldSeparator=X The separator to use to separate fields in intermediate" + "\n\t files. Defaults to 'INFORMATION SEPARATOR ONE' (U+001F)." + "\n\t\t--silent Output only the data." + "\n\t\t--i Make search case insensitive." + "\n\t\t--a Enable AND searching." + "\n\t\t--log Save all the logs.\n" + ";****************************************"); System.exit(1); } //Parse time inputs for start and end of search args[startNum] = tools.parseDate(args[startNum]); args[endNum] = tools.parseDate(args[endNum]); tools.checkTime(args[startNum], args[endNum]); //Retrieve 'strings' argument to be able to pass search strings to HDFS //Retrieve 'out' argument to determine where output of results should be sent String strings = args[stringsNum]; String out = args[outNum]; //Generate files to temporarily store output of mapreduce jobs and pig logs locally File local_output = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { local_output.deleteOnExit(); } File pig_tmp = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { pig_tmp.deleteOnExit(); } //Name the temp directory for storing results in HDFS String tmp = "tmp/logmultisearch-" + RandomStringUtils.randomAlphanumeric(10); //Set args[stringsNum] to be location of search strings to be used for the Multisearch args[stringsNum] = (StringEscapeUtils.escapeJava(tmp) + "/strings"); //Set args[outNum] to be temp output directory to be passed onto MultiSearchByTime instead of UserInput argument args[outNum] = (StringEscapeUtils.escapeJava(tmp) + "/rawlines"); //Managing console output - deal with --v/--silent Logger LOG = LoggerFactory.getLogger(logmultisearch.class); tools.setConsoleOutput(local_output, quiet, silent); //Create temp directory in HDFS to store logsearch logs before sorting tools.tmpDirHDFS(quiet, silent, fs, conf, tmp, log); //If the strings argument is the path of a file, copy the file to HDFS. //If the strings argument is the path of a directory, copy all files in the directory to HDFS. //If the strings argument is not a path to a file/directory, write to a newly created file in HDFS. try { File f = new File(strings); if (f.isFile()) { LogTools.logConsole(quiet, silent, warn, "Strings input is a File..."); //dos2unix file conversion File dos2unix = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); dos2unix.deleteOnExit(); tools.dosTounix(f, dos2unix); //Copy over temp directory into a new directory in HDFS to be used for logmultisearch fs.copyFromLocalFile(new Path(dos2unix.getAbsolutePath()), new Path(tmp + "/strings")); } else if (f.isDirectory()) { LogTools.logConsole(quiet, silent, warn, "Strings input is a Directory..."); //Get list of all files in directory to convert from dos2unix String[] fileList = f.list(); //Create temp directory to store all converted files File tempDir = Files.createTempDir(); tempDir.deleteOnExit(); //Convert all files from dos2unix and write to temp directory for (int i = 0; i < fileList.length; i++) { File dos2unix = File.createTempFile("unix", fileList[i], tempDir); dos2unix.deleteOnExit(); tools.dosTounix(new File(f.getAbsolutePath() + "/" + fileList[i]), dos2unix); } //Copy over temp directory into a new directory in HDFS to be used for logmultisearch fs.copyFromLocalFile(new Path(tempDir.getAbsolutePath()), new Path(tmp + "/strings")); } else { LogTools.logConsole(quiet, silent, warn, "Strings input is a search string..."); //Make directory and file for strings fs.mkdirs(new Path(tmp + "/strings")); fs.createNewFile(new Path(tmp + "/strings/strings")); //Write search strings to file FSDataOutputStream hdfsOut = fs.create(new Path(tmp + "/strings/strings")); hdfsOut.writeUTF(strings); hdfsOut.close(); } } catch (Exception e) { e.printStackTrace(); System.exit(1); } LogTools.logConsole(quiet, silent, warn, "Searching..."); LogTools.logConsole(quiet, silent, warn, "Passing Arguments: SearchStrings=" + strings + " DC=" + args[dcNum] + " Service=" + args[svcNum] + " Component=" + args[compNum] + " StartTime=" + args[startNum] + " EndTime=" + args[endNum] + " Output=" + out); //Set standard configuration for running Mapreduce and PIG String queue_name = "logsearch"; //Start Mapreduce job tools.runMRJob(quiet, silent, conf, D_options, out, LOG, field_separator, queue_name, args, "MultiSearchByTime", new MultiSearchByTime()); //Before sorting, determine the number of records and size of the results found long foundresults = tools.getResults(local_output); long size = tools.getSize(foundresults, tmp, fs); //Run PIG job if results found tools.runPig(silent, quiet, foundresults, size, tmp, out, D_options, queue_name, date_format, field_separator, pig_tmp, fs, conf, forcelocal, forceremote); //Display location of tmp files if log enabled tools.logs(log, local_output, pig_tmp, tmp); return 0; }
From source file:com.blackberry.logtools.logsearch.java
License:Apache License
public int run(String[] argv) throws Exception { //Configuring configuration and filesystem to work on HDFS final Configuration conf = getConf(); //Configuration processed by ToolRunner FileSystem fs = FileSystem.get(conf); //Initiate tools used for running search LogTools tools = new LogTools(); //Other options String date_format = "RFC5424"; String field_separator = " "; ArrayList<String> D_options = new ArrayList<String>(); boolean quiet = true; boolean silent = false; boolean log = false; boolean forcelocal = false; boolean forceremote = false; //The arguments are // - search string // - dc number // - service/*from w ww. jav a 2 s .co m*/ // - component // - startTime (Something 'date' can parse, or just a time in ms from epoch) // - endTime (Same as start) // - outputDir //Indexing for arguments to be passed for Mapreduce int stringNum = 0; int dcNum = 1; int svcNum = 2; int compNum = 3; int startNum = 4; int endNum = 5; int outNum = 6; //Parsing through user arguments String[] args = new String[7]; int count = 0; //Count created to track the parse of all arguments int argcount = 0; //Count created to track number of arguments to be passed on while (count < argv.length) { String arg = argv[count]; count++; if (arg.equals("--")) { break; } else if (arg.startsWith("-")) { if (arg.equals("--v")) { quiet = tools.parseV(silent); } else if (arg.equals("--i")) { conf.set("logdriver.search.case.insensitive", "true"); } else if (arg.startsWith("--dateFormat=")) { arg = arg.replace("--dateFormat=", ""); date_format = arg; } else if (arg.startsWith("--fieldSeparator=")) { arg = arg.replace("--fieldSeparator=", ""); field_separator = arg; } else if (arg.startsWith("-string=")) { arg = arg.replace("-string=", ""); args[stringNum] = arg; argcount++; } else if (arg.startsWith("-dc=")) { arg = arg.replace("-dc=", ""); args[dcNum] = arg; argcount++; } else if (arg.startsWith("-svc=")) { arg = arg.replace("-svc=", ""); args[svcNum] = arg; argcount++; } else if (arg.startsWith("-comp=")) { arg = arg.replace("-comp=", ""); args[compNum] = arg; argcount++; } else if (arg.startsWith("-start=")) { arg = arg.replace("-start=", ""); args[startNum] = arg; argcount++; } else if (arg.startsWith("-end=")) { arg = arg.replace("-end=", ""); args[endNum] = arg; argcount++; } //User inputs output directory that is to be created //Check to see if parent directory exists && output directory does not exist else if (arg.startsWith("--out=")) { args[outNum] = tools.parseOut(arg, fs); argcount++; } else if (arg.startsWith("-D")) { D_options.add(arg); } else if (arg.equals("--silent")) { silent = tools.parseSilent(quiet); } else if (arg.equals("--log")) { log = true; } else if (arg.equals("--l")) { forcelocal = tools.parsePigMode(forceremote); } else if (arg.equals("--r")) { forceremote = tools.parsePigMode(forcelocal); } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } else { LogTools.logConsole(quiet, silent, error, "Unrecognized option: " + arg); System.exit(1); } } //Default output should be stdout represented by "-" if (args[outNum] == null) { args[outNum] = "-"; argcount++; LogTools.logConsole(quiet, silent, info, "Output set to default stdout."); } if (argcount < 7) { System.err.println(";****************************************" + "\n\t\t\t NOT ENOUGH ARGUMENTS\n" + "\n\tUSAGE: logsearch [REQUIRED ARGUMENTS] [OPTIONS] (Order does not matter)" + "\n\tREQUIRED ARGUMENTS:" + "\n\t\t-string=[STRING] String to search." + "\n\t\t-dc=[DATACENTER] Data Center." + "\n\t\t-svc=[SERVICE] Service." + "\n\t\t-comp=[COMPONENT] Component." + "\n\t\t-start=[START] Start time." + "\n\t\t-end=[END] End time." + "\n\tOptions:" + "\n\t\t--out=[DIRECTORY] Desired output directory. If not defined, output to stdout." + "\n\t\t--v Verbose output." + "\n\t\t--r Force remote sort." + "\n\t\t--l Force local sort." + "\n\t\t--dateFormat=[FORMAT] Valid formats are RFC822, RFC3164 (zero padded day)," + "\n\t RFC5424 (default), or any valid format string for FastDateFormat." + "\n\t\t--fieldSeparator=X The separator to use to separate fields in intermediate" + "\n\t files. Defaults to 'INFORMATION SEPARATOR ONE' (U+001F)." + "\n\t\t--silent Output only the data." + "\n\t\t--i Make search case insensitive." + "\n\t\t--log Save all the logs.\n" + ";****************************************"); System.exit(1); } //Parse time inputs for start and end of search args[startNum] = tools.parseDate(args[startNum]); args[endNum] = tools.parseDate(args[endNum]); tools.checkTime(args[startNum], args[endNum]); //Retrieve 'out' argument to determine where output of results should be sent String out = args[outNum]; //Generate files to temporarily store output of mapreduce jobs and pig logs locally File local_output = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { local_output.deleteOnExit(); } File pig_tmp = File.createTempFile("tmp.", RandomStringUtils.randomAlphanumeric(10)); if (log != true) { pig_tmp.deleteOnExit(); } //Name the temp directory for storing results in HDFS String tmp = "tmp/logsearch-" + RandomStringUtils.randomAlphanumeric(10); //Set args[outNum] to be temp output directory to be passed onto FastSearchByTime instead of UserInput argument args[outNum] = (StringEscapeUtils.escapeJava(tmp) + "/rawlines"); //Managing console output - deal with --v/--silent Logger LOG = LoggerFactory.getLogger(logsearch.class); tools.setConsoleOutput(local_output, quiet, silent); //Create temp directory in HDFS to store logsearch logs before sorting tools.tmpDirHDFS(quiet, silent, fs, conf, tmp, log); LogTools.logConsole(quiet, silent, warn, "Searching for " + args[stringNum] + "..."); LogTools.logConsole(quiet, silent, warn, "Passing Arguments: SearchString=" + args[stringNum] + " DC=" + args[dcNum] + " Service=" + args[svcNum] + " Component=" + args[compNum] + " StartTime=" + args[startNum] + " EndTime=" + args[endNum] + " Output=" + out); //Set standard configuration for running Mapreduce and PIG String queue_name = "logsearch"; //Start Mapreduce job tools.runMRJob(quiet, silent, conf, D_options, out, LOG, field_separator, queue_name, args, "FastSearchByTime", new FastSearchByTime()); //Before sorting, determine the number of records and size of the results found long foundresults = tools.getResults(local_output); long size = tools.getSize(foundresults, tmp, fs); //Run PIG job if results found tools.runPig(silent, quiet, foundresults, size, tmp, out, D_options, queue_name, date_format, field_separator, pig_tmp, fs, conf, forcelocal, forceremote); //Display location of tmp files if log enabled tools.logs(log, local_output, pig_tmp, tmp); return 0; }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public void runMRJob(boolean quiet, boolean silent, Configuration conf, ArrayList<String> D_options, String out, Logger LOG, String field_separator, String queue_name, String[] args, String job, Tool tool) throws Exception { logConsole(quiet, silent, info, "Running Mapreduce job & Calling " + job); if (out.equals("-")) { //Uncompress results to be able to read to stdout D_options.add("-Dmapreduce.output.fileoutputformat.compress=false"); }/*from w ww . ja v a 2 s. c o m*/ try { conf.set("zk.connect.string", System.getenv("ZK_CONNECT_STRING")); conf.setBoolean("mapreduce.output.fileoutputformat.compress", true); conf.set("mapred.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); conf.setInt("mapred.max.split.size", 256 * 1024 * 1024); conf.set("logdriver.output.field.separator", field_separator); conf.set("mapred.job.queue.name", StringEscapeUtils.escapeJava(queue_name)); dOpts(D_options, silent, out, conf); //Now run JOB and send arguments LOG.info("Sending args to " + job + ": {}", args); ToolRunner.run(conf, tool, args); } catch (IOException e) { if (e.toString().contains("Failed to find any Kerberos")) { logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate."); System.exit(1); } else if (e.toString().contains("Permission denied")) { logConsole(true, true, error, "Permission denied."); System.err.println("; Please go to https://go/itforms and filled out the Hadoop Onboarding Form " + "to get access to the requested data. Paste the following data into the ticket to help with your request:\n" + "Error Message" + e); System.exit(1); } else if (e.toString().contains("quota") && e.toString().contains("exceeded")) { logConsole(true, true, error, "Disk quota Exceeded."); System.exit(1); } logConsole(true, true, error, "\n\tError running mapreduce job." + generalError() + "\n\tCommand stopped"); e.printStackTrace(); System.exit(1); } }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public void runPigRemote(Map<String, String> params, String out, String tmp, boolean quiet, boolean silent, Configuration conf, String queue_name, String additional_jars, File pig_tmp, ArrayList<String> D_options, String PIG_DIR, FileSystem fs) { //Set input parameter for pig job - calling Pig directly params.put("tmpdir", StringEscapeUtils.escapeJava(tmp)); //Check for an out of '-', meaning write to stdout String pigout;/*from w w w . j av a 2 s .c o m*/ if (out.equals("-")) { params.put("out", tmp + "/final"); pigout = tmp + "/final"; } else { params.put("out", StringEscapeUtils.escapeJava(out)); pigout = StringEscapeUtils.escapeJava(out); } try { logConsole(quiet, silent, info, "Running PIG Command"); conf.set("mapred.job.queue.name", queue_name); conf.set("pig.additional.jars", additional_jars); conf.set("pig.exec.reducers.bytes.per.reducer", Integer.toString(100 * 1000 * 1000)); conf.set("pig.logfile", pig_tmp.toString()); conf.set("hadoopversion", "23"); //PIG temp directory set to be able to delete all temp files/directories conf.set("pig.temp.dir", tmp); //Setting output separator for logdriver String DEFAULT_OUTPUT_SEPARATOR = "\t"; Charset UTF_8 = Charset.forName("UTF-8"); String outputSeparator = conf.get("logdriver.output.field.separator", DEFAULT_OUTPUT_SEPARATOR); byte[] bytes = outputSeparator.getBytes(UTF_8); if (bytes.length != 1) { logConsole(true, true, error, "The output separator must be a single byte in UTF-8."); System.exit(1); } conf.set("logdriver.output.field.separator", Byte.toString(bytes[0])); dOpts(D_options, silent, out, conf); PigServer pigServer = new PigServer(ExecType.MAPREDUCE, conf); pigServer.registerScript(PIG_DIR + "/formatAndSort.pg", params); } catch (Exception e) { e.printStackTrace(); System.exit(1); } logConsole(quiet, silent, warn, "PIG Job Completed."); if (out.equals("-")) { System.out.println(";#################### DATA RESULTS ####################"); try { //Create filter to find files with the results from PIG job PathFilter filter = new PathFilter() { public boolean accept(Path file) { return file.getName().contains("part-"); } }; //Find the files in the directory, open and printout results FileStatus[] status = fs.listStatus(new Path(tmp + "/final"), filter); for (int i = 0; i < status.length; i++) { BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(status[i].getPath()))); String line; line = br.readLine(); while (line != null) { System.out.println(line); line = br.readLine(); } } System.out.println(";#################### END OF RESULTS ####################"); } catch (IOException e) { e.printStackTrace(); System.exit(1); } } else { System.out.println( ";#################### Done. Search results are in " + pigout + " ####################"); } }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public void runPigLocal(Map<String, String> params, String out, String tmp, final boolean quiet, final boolean silent, Configuration conf, String queue_name, String additional_jars, File pig_tmp, ArrayList<String> D_options, String PIG_DIR, FileSystem fs) throws IllegalArgumentException, IOException { //Create temp file on local to hold data to sort final File local_tmp = Files.createTempDir(); local_tmp.deleteOnExit();//from w ww .jav a 2s. com Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { @Override public void run() { try { logConsole(quiet, silent, warn, "Deleting tmp files in local tmp"); delete(local_tmp); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } })); //Set input parameter for pig job params.put("tmpdir", local_tmp.toString() + "/" + tmp); //Check for an out of '-', meaning write to stdout String pigout; if (out.equals("-")) { params.put("out", local_tmp + "/" + tmp + "/final"); pigout = local_tmp + "/" + tmp + "/final"; } else { params.put("out", local_tmp + "/" + StringEscapeUtils.escapeJava(out)); pigout = StringEscapeUtils.escapeJava(out); } //Copy the tmp folder from HDFS to the local tmp directory, and delete the remote folder fs.copyToLocalFile(true, new Path(tmp), new Path(local_tmp + "/" + tmp)); try { logConsole(quiet, silent, info, "Running PIG Command"); conf.set("mapred.job.queue.name", queue_name); conf.set("pig.additional.jars", additional_jars); conf.set("pig.exec.reducers.bytes.per.reducer", Integer.toString(100 * 1000 * 1000)); conf.set("pig.logfile", pig_tmp.toString()); conf.set("hadoopversion", "23"); //PIG temp directory set to be able to delete all temp files/directories conf.set("pig.temp.dir", local_tmp.getAbsolutePath()); //Setting output separator for logdriver String DEFAULT_OUTPUT_SEPARATOR = "\t"; Charset UTF_8 = Charset.forName("UTF-8"); String outputSeparator = conf.get("logdriver.output.field.separator", DEFAULT_OUTPUT_SEPARATOR); byte[] bytes = outputSeparator.getBytes(UTF_8); if (bytes.length != 1) { System.err.println( ";******************** The output separator must be a single byte in UTF-8. ******************** "); System.exit(1); } conf.set("logdriver.output.field.separator", Byte.toString(bytes[0])); dOpts(D_options, silent, out, conf); PigServer pigServer = new PigServer(ExecType.LOCAL, conf); UserGroupInformation.setConfiguration(new Configuration(false)); pigServer.registerScript(PIG_DIR + "/formatAndSortLocal.pg", params); } catch (Exception e) { e.printStackTrace(); System.exit(1); } logConsole(quiet, silent, warn, "PIG Job Completed."); if (out.equals("-")) { System.out.println(";#################### DATA RESULTS ####################"); try { File results = new File(pigout); String[] resultList = results.list(); //Find the files in the directory, open and printout results for (int i = 0; i < resultList.length; i++) { if (resultList[i].contains("part-") && !resultList[i].contains(".crc")) { BufferedReader br = new BufferedReader( new FileReader(new File(pigout + "/" + resultList[i]))); String line; line = br.readLine(); while (line != null) { System.out.println(line); line = br.readLine(); } br.close(); } } System.out.println(";#################### END OF RESULTS ####################"); } catch (IOException e) { e.printStackTrace(); System.exit(1); } } else { fs.copyFromLocalFile(new Path(local_tmp + "/" + StringEscapeUtils.escapeJava(out)), new Path(pigout)); System.out.println( ";#################### Done. Search results are in " + pigout + " ####################"); } }
From source file:com.blackberry.logtools.LogTools.java
License:Apache License
public static void dOpts(ArrayList<String> D_options, boolean silent, String out, Configuration conf) { //Run overriden D_options if (D_options.size() != 0) { if (!silent && !(out.equals("-") && D_options.size() == 1)) { System.err.println("; Running overriden -D options:"); }/* w ww . j ava 2s .c o m*/ for (int opt = 0; opt < D_options.size(); opt++) { String[] pig_opts = new String[2]; pig_opts = (D_options.get(opt)).split("="); conf.set(pig_opts[0].replace("-D", ""), pig_opts[1]); } } }
From source file:com.boozallen.cognition.ingest.storm.util.HdfsFileLoader.java
License:Apache License
private FileSystem getHadoopFileSystem() { Configuration conf = new Configuration(); if (_hadoopConfig.isEmpty()) { conf.addResource(new Path(_hadoopConfDirectory + File.separator + "core-site.xml")); conf.addResource(new Path(_hadoopConfDirectory + File.separator + "hdfs-site.xml")); } else {/* w w w . j ava 2 s. c o m*/ for (Map.Entry<String, String> entry : _hadoopConfig.entrySet()) { conf.set(entry.getKey(), entry.getValue()); } } try { return FileSystem.get(conf); } catch (IOException e) { throw new RuntimeException(e); } }
From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.PFPGrowth.java
License:Apache License
/** * @throws ClassNotFoundException // w w w .j a v a 2s .c o m * @throws InterruptedException * @throws IOException * @params * input, output locations, additional parameters like minSupport(3), maxHeapSize(50), numGroups(1000) * @conf * initial Hadoop configuration to use. * * */ public static void runPFPGrowth(Parameters params, Configuration conf) throws IOException, InterruptedException, ClassNotFoundException { conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); startParallelCounting(params, conf); // save feature list to dcache List<Pair<String, Long>> fList = readFList(params); saveFList(fList, params, conf); // set param to control group size in MR jobs int numGroups = params.getInt(NUM_GROUPS, NUM_GROUPS_DEFAULT); int maxPerGroup = fList.size() / numGroups; if (fList.size() % numGroups != 0) { maxPerGroup++; } params.set(MAX_PER_GROUP, Integer.toString(maxPerGroup)); startParallelFPGrowth(params, conf); //startAggregating(params, conf); }
From source file:com.cg.mapreduce.fpgrowth.mahout.fpm.PFPGrowth.java
License:Apache License
/** * Run the aggregation Job to aggregate the different TopK patterns and group each Pattern by the features * present in it and thus calculate the final Top K frequent Patterns for each feature *//*from w w w. j a v a 2 s . c o m*/ public static void startAggregating(Parameters params, Configuration conf) throws IOException, InterruptedException, ClassNotFoundException { conf.set(PFP_PARAMETERS, params.toString()); conf.set("mapred.compress.map.output", "true"); conf.set("mapred.output.compression.type", "BLOCK"); Path input = new Path(params.get(OUTPUT), FPGROWTH); Job job = new Job(conf, "PFP Aggregator Driver running over input: " + input); job.setJarByClass(PFPGrowth.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(TopKStringPatterns.class); FileInputFormat.addInputPath(job, input); Path outPath = new Path(params.get(OUTPUT), FREQUENT_PATTERNS); FileOutputFormat.setOutputPath(job, outPath); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(AggregatorMapper.class); job.setCombinerClass(AggregatorReducer.class); job.setReducerClass(AggregatorReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); HadoopUtil.delete(conf, outPath); boolean succeeded = job.waitForCompletion(true); if (!succeeded) { throw new IllegalStateException("Job failed!"); } }