Java tutorial
/** * @UNCC Fodor Lab * @author Michael Sioda * @email msioda@uncc.edu * @date Feb 9, 2017 * @disclaimer This code is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version, * provided that any use properly credits the author. * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details at http://www.gnu.org * */ package bioLockJ.module.agent; import java.io.BufferedReader; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.Base64; import java.util.Collection; import java.util.List; import java.util.Properties; import javax.activation.DataHandler; import javax.activation.DataSource; import javax.activation.FileDataSource; import javax.crypto.Cipher; import javax.crypto.SecretKey; import javax.crypto.SecretKeyFactory; import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEParameterSpec; import javax.mail.Authenticator; import javax.mail.BodyPart; import javax.mail.Message; import javax.mail.Multipart; import javax.mail.PasswordAuthentication; import javax.mail.Session; import javax.mail.Transport; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeBodyPart; import javax.mail.internet.MimeMessage; import javax.mail.internet.MimeMultipart; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.commons.configuration.PropertiesConfigurationLayout; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.lang.math.NumberUtils; import bioLockJ.AppController; import bioLockJ.Config; import bioLockJ.Constants; import bioLockJ.Log; import bioLockJ.Module; import bioLockJ.PropertyFileContainer; import bioLockJ.util.r.RScript; /** * A simple Mail utility to send notifications when job is complete with a status message * and summary details about failures and run time. */ public class MailAgent extends Module { @Override public void checkDependencies() throws Exception { emailTos = Config.requireList(EMAIL_TO); emailHost = Config.requireString(EMAIL_HOST); emailPort = Config.requireString(EMAIL_PORT); emailSmtpAuth = Config.requireString(EMAIL_SMTP_AUTH); emailTls = Config.requireString(EMAIL_START_TLS_ENABLE); emailFrom = Config.requireString(EMAIL_FROM); emailEncryptedPassword = Config.requireString(EMAIL_ENCRYPTED_PASSWORD); emailMaxAttachmentMB = Config.requirePositiveInteger(EMAIL_ATTACHMENT_MAX_MB); emailIncludeQsub = Config.getBoolean(EMAIL_SEND_QSUB); clusterHost = Config.getString(CLUSTER_HOST); new InternetAddress(emailFrom).validate(); for (final String email : emailTos) { new InternetAddress(email).validate(); } } @Override public void executeProjectFile() throws Exception { logFailures(); status = getStatus(); Transport.send(getMimeMessage()); Log.out.info("EMAIL SENT!"); } private String decrypt(final String property) { String decryptedPassword = null; try { final SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndDES"); final SecretKey key = keyFactory.generateSecret(new PBEKeySpec(PASSWORD)); final Cipher pbeCipher = Cipher.getInstance("PBEWithMD5AndDES"); pbeCipher.init(Cipher.DECRYPT_MODE, key, new PBEParameterSpec(SALT, 20)); decryptedPassword = new String(pbeCipher.doFinal(base64Decode(property)), "UTF-8"); } catch (final Exception ex) { Log.out.error(ex.getMessage(), ex); } return decryptedPassword; } private BodyPart getAttachment(final String filePath) throws Exception { try { final DataSource source = new FileDataSource(filePath); final File logFile = new File(filePath); final double fileSize = logFile.length() / 1000000; if (fileSize < emailMaxAttachmentMB) { final BodyPart attachPart = new MimeBodyPart(); attachPart.setDataHandler(new DataHandler(source)); attachPart.setFileName(filePath.substring(filePath.lastIndexOf(File.separator) + 1)); return attachPart; } else { Log.out.warn("File [" + filePath + "] too large to attach. Max file size configured in prop file set to = " + emailMaxAttachmentMB + " MB"); } } catch (final Exception ex) { Log.out.error("Unable to attach file", ex); } return null; } private List<BodyPart> getAttachments() throws Exception { final List<BodyPart> attachments = new ArrayList<>(); if (emailIncludeQsub) { final List<BodyPart> qsubLogs = getQsubLogs(); if (qsubLogs != null) { attachments.addAll(getQsubLogs()); } } return attachments; } private Multipart getContent() throws Exception { final Multipart multipart = new MimeMultipart(); final BodyPart messageBodyPart = new MimeBodyPart(); final List<BodyPart> attachments = getAttachments(); messageBodyPart.setText(getSummary()); multipart.addBodyPart(messageBodyPart); for (final BodyPart bp : attachments) { multipart.addBodyPart(bp); } return multipart; } /** * Example * scp -r -p msioda@hpc.uncc.edu:/users/msioda/data/metagenomics/* /Users/msioda/projects/downloads * * @return download msg */ private String getDownloadMsg() throws Exception { String user = System.getProperty("user.home"); final String d = getDownloadDir(); if (!previousModule.isComplete() || user.isEmpty() || clusterHost.isEmpty() || (clusterHost == null) || (user == null) || (d == null)) { return Constants.RETURN + Constants.RETURN + "Project directory: " + Config.requireExistingDirectory(Config.PROJECT_DIR).getAbsolutePath() + Constants.RETURN + Constants.RETURN; } final File logFileCopy = new File(getInputDir().getAbsolutePath() + File.separator + Log.getName()); FileUtils.copyFile(Log.getFile(), logFileCopy); final int index = user.lastIndexOf(File.separator); if (index > 0) { user = user.substring(index + 1); } return Constants.RETURN + Constants.RETURN + "Run the following command to download results:" + Constants.RETURN + Constants.RETURN + Constants.TAB_DELIM + Constants.TAB_DELIM + "scp -r -p " + user + "@" + clusterHost + ":" + getInputDir().getAbsolutePath() + " " + d + Constants.RETURN + Constants.RETURN; } private int getMaxModuleNameLength() throws Exception { int maxModuleNameLen = TOTAL_RUNTIME_LABEL.length(); for (final Module m : Config.getModules()) { if (!(m instanceof MailAgent) && (m.getClass().getSimpleName().length() > maxModuleNameLen)) { maxModuleNameLen = m.getClass().getSimpleName().length(); } } return maxModuleNameLen; } private Message getMimeMessage() throws Exception { final Message message = new MimeMessage(getSession()); message.setFrom(new InternetAddress(emailFrom)); message.addRecipients(Message.RecipientType.TO, InternetAddress.parse(getRecipients())); message.setSubject("BioLockJ " + Config.requireString(Config.PROJECT_NAME) + " " + status); message.setContent(getContent()); return message; } private List<BodyPart> getQsubLogs() throws Exception { final List<BodyPart> attachments = new ArrayList<>(); final List<String> qsubDirs = Config.getList(Config.QSUBS); for (final String qsub : qsubDirs) { final Collection<File> files = FileUtils.listFiles(new File(qsub), TrueFileFilter.INSTANCE, null); for (final File f : files) { final BodyPart qsubAttachment = getAttachment(f.getAbsolutePath()); if (qsubAttachment != null) { attachments.add(qsubAttachment); } else { Log.out.warn("Unable to attache Qsub output: " + f.getAbsolutePath()); } } } return attachments; } private String getRecipients() { final StringBuffer addys = new StringBuffer(); for (final String to : emailTos) { if (addys.length() != 0) { addys.append(","); } addys.append(to); } return addys.toString(); } private Session getSession() { final Properties props = new Properties(); props.put(EMAIL_SMTP_AUTH, emailSmtpAuth); props.put(EMAIL_START_TLS_ENABLE, emailTls); props.put(EMAIL_HOST, emailHost); props.put(EMAIL_PORT, emailPort); final Session session = Session.getInstance(props, new Authenticator() { @Override protected PasswordAuthentication getPasswordAuthentication() { return new PasswordAuthentication(emailFrom, decrypt(emailEncryptedPassword)); } }); return session; } private String getStatus() throws Exception { final File[] dirs = Config.requireExistingDirectory(Config.PROJECT_DIR) .listFiles((FileFilter) DirectoryFileFilter.DIRECTORY); for (final File d : dirs) { final String prefix = d.getName().substring(0, 1); if (NumberUtils.isNumber(prefix) && !d.getName().contains(MailAgent.class.getSimpleName())) { if (FileUtils.directoryContains(d, new File(d.getAbsolutePath() + File.separator + Constants.BLJ_STARTED))) { failedModule = d.getName(); Log.out.warn("Module Failed: " + failedModule); return Constants.FAILED; } } } return Constants.SUCCESS; } /** * Summary of run-times to be output to output file & to be included in summary email. * @return */ private String getSummary() throws Exception { final StringBuffer sb = new StringBuffer(); sb.append("BioLockJ [" + Config.requireString(Config.PROJECT_NAME) + "] " + " pipeline " + status.toLowerCase() + "!" + getDownloadMsg()); Log.out.info(Constants.LOG_SPACER); Log.out.info("Program " + status + "!"); Log.out.info(Constants.LOG_SPACER); if (failedModule != null) { sb.append(Constants.INDENT + "Failed Module: " + failedModule + Constants.RETURN); if (failedModule.equals(RScriptAgent.class.getSimpleName())) { boolean found = false; for (final File f : getInputFiles()) { if (f.getName().endsWith(RScript.R_ERROR)) { found = true; sb.append( Constants.INDENT + "------------------------------------------" + Constants.RETURN); sb.append(Constants.INDENT + "R Script Failures Detected!" + Constants.RETURN); final BufferedReader r = AppController.getFileReader(f); for (String line = r.readLine(); line != null; line = r.readLine()) { sb.append(Constants.INDENT + line + Constants.RETURN); } sb.append( Constants.INDENT + "------------------------------------------" + Constants.RETURN); } } if (found) { sb.append(Constants.RETURN); } } } for (final File f : getFailures()) { sb.append(Constants.INDENT + f.getName() + Constants.RETURN); } sb.append(Constants.RETURN + SPACER + Constants.RETURN); final int max = getMaxModuleNameLength(); for (final Module m : Config.getModules()) { if (!(m instanceof MailAgent)) { final String runTime = getLabel(m.getClass().getSimpleName(), max) + m.getRunTime(); sb.append(runTime + Constants.RETURN); Log.out.info(runTime); } } final String runTime = getLabel(TOTAL_RUNTIME_LABEL, max) + AppController.getRunTime(); Log.out.info(Constants.LOG_SPACER); Log.out.info(runTime); Log.out.info(Constants.LOG_SPACER); sb.append(SPACER + Constants.RETURN); sb.append(runTime + Constants.RETURN); sb.append(SPACER + Constants.RETURN); sb.append(Constants.RETURN + Constants.RETURN + "Regards," + Constants.RETURN + Constants.RETURN + "BioLockJ Admin"); return sb.toString(); } /** * Used to obtain a new encrypted password hash when the admin email password is set. * @param password * @throws Exception */ public static void encryptAndStoreEmailPassword(final String propFile, final String password) throws Exception { final String encryptedPassword = encrypt(password); final PropertiesConfigurationLayout layout = new PropertiesConfigurationLayout( new PropertiesConfiguration()); layout.load(new InputStreamReader(new FileInputStream(propFile))); final PropertyFileContainer props = Config.readProps(new File(propFile), null); props.setProperty(EMAIL_ENCRYPTED_PASSWORD, encryptedPassword); layout.save(new FileWriter(propFile, false)); System.out.println("CONFIG FILE UPDATED WITH ENCRYPTED PASSWORD: " + encryptedPassword); } private static byte[] base64Decode(final String encodedPassword) throws IOException { return Base64.getDecoder().decode(encodedPassword); } private static String base64Encode(final byte[] bytes) { return Base64.getEncoder().encodeToString(bytes); } private static String encrypt(final String password) throws GeneralSecurityException, UnsupportedEncodingException { final SecretKeyFactory keyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndDES"); final SecretKey key = keyFactory.generateSecret(new PBEKeySpec(PASSWORD)); final Cipher pbeCipher = Cipher.getInstance("PBEWithMD5AndDES"); pbeCipher.init(Cipher.ENCRYPT_MODE, key, new PBEParameterSpec(SALT, 20)); return base64Encode(pbeCipher.doFinal(password.getBytes("UTF-8"))); } private static String getLabel(final String label, final int max) { final int bufferSize = max - label.length(); String buffer = ""; while (buffer.length() < bufferSize) { buffer = buffer + " "; } return label + ":" + buffer + Constants.INDENT; } private String clusterHost = null; private String emailEncryptedPassword = null; private String emailFrom = null; private String emailHost = null; private boolean emailIncludeQsub = false; private int emailMaxAttachmentMB = 0; private String emailPort = null; private String emailSmtpAuth = null; private String emailTls = null; private List<String> emailTos = null; private String failedModule = null; private String status = null; protected static final String CLUSTER_HOST = "cluster.host"; protected static final String EMAIL_ATTACHMENT_MAX_MB = "mail.maxAttachmentSizeMB"; protected static final String EMAIL_ENCRYPTED_PASSWORD = "mail.encryptedPassword"; protected static final String EMAIL_FROM = "mail.from"; protected static final String EMAIL_HOST = "mail.smtp.host"; protected static final String EMAIL_PORT = "mail.smtp.port"; protected static final String EMAIL_SEND_QSUB = "mail.sendQsub"; protected static final String EMAIL_SMTP_AUTH = "mail.smtp.auth"; protected static final String EMAIL_START_TLS_ENABLE = "mail.smtp.starttls.enable"; protected static final String EMAIL_TO = "mail.to"; private static final char[] PASSWORD = "enfldsgbnlsngdlksdsgm".toCharArray(); private static final byte[] SALT = { (byte) 0xde, (byte) 0x33, (byte) 0x10, (byte) 0x12, (byte) 0xde, (byte) 0x33, (byte) 0x10, (byte) 0x12, }; private static final String SPACER = "---------------------------------------------------------------------"; private static final String TOTAL_RUNTIME_LABEL = "Total Runtime"; }