Java tutorial
/** * The contents of this file are subject to the license and copyright * detailed in the LICENSE and NOTICE files at the root of the source * tree and available online at * * http://www.dspace.org/license/ */ package org.dspace.statistics.util; import org.apache.commons.cli.*; import org.apache.log4j.Logger; import org.dspace.app.statistics.LogAnalyser; import org.dspace.app.statistics.LogLine; import org.dspace.content.*; import org.dspace.handle.HandleManager; import org.dspace.core.Context; import java.io.*; import java.sql.SQLException; import java.util.regex.Pattern; import java.util.regex.Matcher; import java.text.SimpleDateFormat; import java.text.ParsePosition; /** * A utility class to convert the classic dspace.log (as generated * by log4j) files into an intermediate format for ingestion into * the new solr stats. * * @author Stuart Lewis */ public class ClassicDSpaceLogConverter { private Logger log = Logger.getLogger(ClassicDSpaceLogConverter.class); /** A DSpace context */ private Context context; /** Whether or not to provide verbose output */ private boolean verbose = false; /** Whether to include actions logged by org.dspace.usage.LoggerUsageEventListener */ private boolean newEvents = false; /** A regular expression for extracting the IP address from a log line */ private Pattern ipaddrPattern = Pattern.compile("ip_addr=(\\d*\\.\\d*\\.\\d*\\.\\d*):"); /** Date format (in) from the log line */ private SimpleDateFormat dateFormatIn = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); /** Date format out (for solr) */ private SimpleDateFormat dateFormatOut = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); /** Date format (in) from the log line for the UID */ private SimpleDateFormat dateFormatInUID = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS"); /** Date format out (for uid) */ private SimpleDateFormat dateFormatOutUID = new SimpleDateFormat("yyyyMMddHHmmssSSS"); /** * Create an instance of the converter utility * * @param c The context * @param v Whether or not to provide verbose output * @param nE Whether to include actions logged by org.dspace.usage.LoggerUsageEventListener */ public ClassicDSpaceLogConverter(Context c, boolean v, boolean nE) { // Set up some variables context = c; verbose = v; newEvents = nE; } /** * Convert a classic log file * * @param in The filename to read from * @param out The filename to write to * @return The number of lines processed */ public int convert(String in, String out) { // Line counter int counter = 0; int lines = 0; // Figure out input, output BufferedReader input; Writer output; try { if (null == in || in.isEmpty() || "-".equals(in)) { input = new BufferedReader(new InputStreamReader(System.in)); in = "standard input"; } else input = new BufferedReader(new FileReader(in)); if (null == out || out.isEmpty() || "-".equals(out)) { output = new BufferedWriter(new OutputStreamWriter(System.out)); out = "standard output"; } else output = new BufferedWriter(new FileWriter(out)); } catch (IOException ie) { log.error("File access problem", ie); return 0; } // Say what we're going to do System.err.println(" About to convert '" + in + "' to '" + out + "'"); // Setup the regular expressions for the log file LogAnalyser.setRegex(in); // Open the file and read it line by line try { String line; LogLine lline; String lout; String id; String handle; String ip; String date; DSpaceObject dso; String uid; String lastLine = ""; while ((line = input.readLine()) != null) { // Read in the line and convert it to a LogLine lines++; if (verbose) { System.out.println(" - IN: " + line); } lline = LogAnalyser.getLogLine(line); // Get rid of any lines that aren't INFO if ((lline == null) || (!lline.isLevel("INFO"))) { if (verbose) { System.out.println(" - IGNORED!"); } continue; } // Get the IP address of the user Matcher matcher = ipaddrPattern.matcher(line); if (matcher.find()) { ip = matcher.group(1); } else { ip = "unknown"; } // Get and format the date // We can use lline.getDate() as this strips the time element date = dateFormatOut .format(dateFormatIn.parse(line.substring(0, line.indexOf(',')), new ParsePosition(0))); // Generate a UID for the log line // - based on the date/time uid = dateFormatOutUID.format(dateFormatInUID .parse(line.substring(0, line.indexOf(' ', line.indexOf(' ') + 1)), new ParsePosition(0))); try { // What sort of view is it? // (ignore lines from org.dspace.usage.LoggerUsageEventListener which is 1.6 code) if ((lline.getAction().equals("view_bitstream")) && (!lline.getParams().contains("invalid_bitstream_id")) && (!lline.getParams().contains("withdrawn")) && ((!line.contains("org.dspace.usage.LoggerUsageEventListener")) || newEvents)) { id = lline.getParams().substring(13); } else if ((lline.getAction().equals("view_item")) && ((!line.contains("org.dspace.usage.LoggerUsageEventListener")) || newEvents)) { handle = lline.getParams().substring(7); dso = HandleManager.resolveToObject(context, handle); id = "" + dso.getID(); } else if ((lline.getAction().equals("view_collection")) && ((!line.contains("org.dspace.usage.LoggerUsageEventListener")) || newEvents)) { id = lline.getParams().substring(14); } else if ((lline.getAction().equals("view_community")) && ((!line.contains("org.dspace.usage.LoggerUsageEventListener")) || newEvents)) { id = lline.getParams().substring(13); } else { //if (verbose) System.out.println(" - IGNORED!"); continue; } // Construct the log line lout = uid + "," + lline.getAction() + "," + id + "," + date + "," + lline.getUser() + "," + ip + "\n"; } catch (Exception e) { if (verbose) { System.out.println(" - IN: " + line); } if (verbose) { System.err.println("Error with log line! " + e.getMessage()); } continue; } if ((verbose) && (!"".equals(lout))) { System.out.println(" - IN: " + line); System.out.println(" - OUT: " + lout); } // Write the output line if ((!"".equals(lout)) && (!lout.equals(lastLine))) { output.write(lout); counter++; lastLine = lout; } } } catch (IOException e) { log.error("File access problem", e); } finally { // Clean up the input and output streams try { input.close(); } catch (IOException e) { log.error(e.getMessage(), e); } try { output.flush(); } catch (IOException e) { log.error(e.getMessage(), e); } try { output.close(); } catch (IOException e) { log.error(e.getMessage(), e); } } // Tell the user what we have done System.err.println(" Read " + lines + " lines and recorded " + counter + " events"); return counter; } /** * Print the help message * * @param options The command line options the user gave * @param exitCode the system exit code to use */ private static void printHelp(Options options, int exitCode) { // print the help message HelpFormatter myhelp = new HelpFormatter(); myhelp.printHelp("ClassicDSpaceLogConverter\n", options); System.err.println("\n\tClassicDSpaceLogConverter -i infilename -o outfilename -v (for verbose output)"); System.exit(exitCode); } /** * Main method to execute the converter * * @param args CLI args */ public static void main(String[] args) { CommandLineParser parser = new PosixParser(); Options options = new Options(); options.addOption("i", "in", true, "source file ('-' or omit for standard input)"); options.addOption("o", "out", true, "destination file or directory ('-' or omit for standard output)"); options.addOption("m", "multiple", false, "treat the input file as having a wildcard ending"); options.addOption("n", "newformat", false, "process new format log lines (1.6+)"); options.addOption("v", "verbose", false, "display verbose output (useful for debugging)"); options.addOption("h", "help", false, "help"); // Parse the command line arguments CommandLine line; try { line = parser.parse(options, args); } catch (ParseException pe) { System.err.println("Error parsing command line arguments: " + pe.getMessage()); System.exit(1); return; } // Did the user ask to see the help? if (line.hasOption('h')) { printHelp(options, 0); } // Whether or not to include event created by org.dspace.usage.LoggerUsageEventListener boolean newEvents = line.hasOption('n'); // Create a copy of the converter Context context = null; try { context = new Context(); context.turnOffAuthorisationSystem(); } catch (SQLException sqle) { System.err.println("Unable to create DSpace context: " + sqle.getMessage()); System.exit(1); } ClassicDSpaceLogConverter converter = new ClassicDSpaceLogConverter(context, line.hasOption('v'), newEvents); // Set up the log analyser try { LogAnalyser.readConfig(); } catch (IOException ioe) { System.err.println("Unable to read config file: " + LogAnalyser.getConfigFile()); System.exit(1); } // Are we converting multiple files? if (line.hasOption('m')) { // Convert all the files final File sample = new File(line.getOptionValue('i')); File dir = sample.getParentFile(); FilenameFilter filter = new FilenameFilter() { public boolean accept(File dir, String name) { return name.startsWith(sample.getName()); } }; String[] children = dir.list(filter); for (String in : children) { System.err.println(in); String out = line.getOptionValue('o') + (dir.getAbsolutePath() + System.getProperty("file.separator") + in) .substring(line.getOptionValue('i').length()); converter.convert(dir.getAbsolutePath() + System.getProperty("file.separator") + in, out); } } else { // Just convert the one file converter.convert(line.getOptionValue('i'), line.getOptionValue('o')); } // Clean everything up context.restoreAuthSystemState(); context.abort(); } }