Java tutorial
/*********************************************************************************************************************** * * Elasticsearch IMAP/Pop3 E-Mail Importer * ========================================== * * Copyright (C) 2014 by Hendrik Saly (http://saly.de) and others. * * Contains (partially) copied code from Jrg Prante's Elasticsearch JDBC river (https://github.com/jprante/elasticsearch-river-jdbc) * *********************************************************************************************************************** * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. * *********************************************************************************************************************** * * $Id:$ * **********************************************************************************************************************/ package de.saly.elasticsearch.importer.imap.support; import java.io.IOException; import java.io.InputStream; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import javax.mail.BodyPart; import javax.mail.Header; import javax.mail.Message; import javax.mail.Message.RecipientType; import javax.mail.MessagingException; import javax.mail.Multipart; import javax.mail.Part; import javax.mail.UIDFolder; import javax.mail.internet.InternetAddress; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.codehaus.jackson.annotate.JsonAnyGetter; import org.codehaus.jackson.map.DeserializationConfig; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.SerializationConfig; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.ESLoggerFactory; import com.sun.mail.pop3.POP3Folder; public class IndexableMailMessage { protected static final ESLogger logger = ESLoggerFactory.getLogger(IndexableMailMessage.class.getName()); public static IndexableMailMessage fromJavaMailMessage(final Message jmm, final boolean withTextContent, final boolean withHtmlContent, final boolean preferHtmlContent, final boolean withAttachments, final boolean stripTags, List<String> headersToFields) throws MessagingException, IOException { final IndexableMailMessage im = new IndexableMailMessage(); @SuppressWarnings("unchecked") final Enumeration<Header> allHeaders = jmm.getAllHeaders(); final Set<IndexableHeader> headerList = new HashSet<IndexableHeader>(); while (allHeaders.hasMoreElements()) { final Header h = allHeaders.nextElement(); headerList.add(new IndexableHeader(h.getName(), h.getValue())); } im.setHeaders(headerList.toArray(new IndexableHeader[headerList.size()])); im.setSelectedHeaders(extractHeaders(im.getHeaders(), headersToFields)); if (jmm.getFolder() instanceof POP3Folder) { im.setPopId(((POP3Folder) jmm.getFolder()).getUID(jmm)); im.setMailboxType("POP"); } else { im.setMailboxType("IMAP"); } if (jmm.getFolder() instanceof UIDFolder) { im.setUid(((UIDFolder) jmm.getFolder()).getUID(jmm)); } im.setFolderFullName(jmm.getFolder().getFullName()); im.setFolderUri(jmm.getFolder().getURLName().toString()); im.setContentType(jmm.getContentType()); im.setSubject(jmm.getSubject()); im.setSize(jmm.getSize()); im.setSentDate(jmm.getSentDate()); if (jmm.getReceivedDate() != null) { im.setReceivedDate(jmm.getReceivedDate()); } if (jmm.getFrom() != null && jmm.getFrom().length > 0) { im.setFrom(Address.fromJavaMailAddress(jmm.getFrom()[0])); } if (jmm.getRecipients(RecipientType.TO) != null) { im.setTo(Address.fromJavaMailAddress(jmm.getRecipients(RecipientType.TO))); } if (jmm.getRecipients(RecipientType.CC) != null) { im.setCc(Address.fromJavaMailAddress(jmm.getRecipients(RecipientType.CC))); } if (jmm.getRecipients(RecipientType.BCC) != null) { im.setBcc(Address.fromJavaMailAddress(jmm.getRecipients(RecipientType.BCC))); } if (withTextContent) { // try { String textContent = getText(jmm, 0, preferHtmlContent); if (stripTags) { textContent = stripTags(textContent); } im.setTextContent(textContent); // } catch (final Exception e) { // logger.error("Unable to retrieve text content for message {} due to {}", // e, ((MimeMessage) jmm).getMessageID(), e); // } } if (withHtmlContent) { // try { String htmlContent = getText(jmm, 0, true); im.setHtmlContent(htmlContent); // } catch (final Exception e) { // logger.error("Unable to retrieve text content for message {} due to {}", // e, ((MimeMessage) jmm).getMessageID(), e); // } } if (withAttachments) { try { final Object content = jmm.getContent(); // look for attachments if (jmm.isMimeType("multipart/*") && content instanceof Multipart) { List<ESAttachment> attachments = new ArrayList<ESAttachment>(); final Multipart multipart = (Multipart) content; for (int i = 0; i < multipart.getCount(); i++) { final BodyPart bodyPart = multipart.getBodyPart(i); if (!Part.ATTACHMENT.equalsIgnoreCase(bodyPart.getDisposition()) && !StringUtils.isNotBlank(bodyPart.getFileName())) { continue; // dealing with attachments only } final InputStream is = bodyPart.getInputStream(); final byte[] bytes = IOUtils.toByteArray(is); IOUtils.closeQuietly(is); attachments.add(new ESAttachment(bodyPart.getContentType(), bytes, bodyPart.getFileName())); } if (!attachments.isEmpty()) { im.setAttachments(attachments.toArray(new ESAttachment[attachments.size()])); im.setAttachmentCount(im.getAttachments().length); attachments.clear(); attachments = null; } } } catch (final Exception e) { logger.error( "Error indexing attachments (message will be indexed but without attachments) due to {}", e, e.toString()); } } im.setFlags(IMAPUtils.toStringArray(jmm.getFlags())); im.setFlaghashcode(jmm.getFlags().hashCode()); return im; } private static Map<String, String> extractHeaders(IndexableHeader[] allHeaders, List<String> headersToFields) { Map<String, String> map = new HashMap<>(); for (String headerName : headersToFields) { for (IndexableHeader header : allHeaders) { // e-mail headers are case insensitive if (headerName.toLowerCase().equals(header.getName().toLowerCase())) { String fieldName = "header_" + headerName.replaceAll("[^A-Za-z0-9 ]", "_").toLowerCase(); map.put(fieldName, header.getValue()); break; } } } return map; } private static String getText(final Part p, int depth, final boolean preferHtmlContent) throws MessagingException, IOException { if (depth >= 100) { throw new IOException("Endless recursion detected "); } // TODO fix encoding for buggy encoding headers if (p.isMimeType("text/*")) { Object content = null; try { content = p.getContent(); } catch (final Exception e) { logger.error("Unable to index the content of a message due to {}", e.toString()); return null; } if (content instanceof String) { final String s = (String) p.getContent(); return s; } if (content instanceof InputStream) { final InputStream in = (InputStream) content; // TODO guess encoding with // http://code.google.com/p/juniversalchardet/ final String s = IOUtils.toString(in, "UTF-8"); IOUtils.closeQuietly(in); return s; } throw new MessagingException("Unknown content class representation: " + content.getClass()); } if (p.isMimeType("multipart/alternative")) { // prefer plain text over html text final Multipart mp = (Multipart) p.getContent(); String text = null; for (int i = 0; i < mp.getCount(); i++) { final Part bp = mp.getBodyPart(i); if (bp.isMimeType("text/html")) { if (text == null) { text = getText(bp, ++depth, preferHtmlContent); } if (preferHtmlContent) { return text; } else { continue; } } else if (bp.isMimeType("text/plain")) { final String s = getText(bp, ++depth, preferHtmlContent); if (s != null && !preferHtmlContent) { return s; } else { continue; } } else { return getText(bp, ++depth, preferHtmlContent); } } return text; } else if (p.isMimeType("multipart/*")) { final Multipart mp = (Multipart) p.getContent(); for (int i = 0; i < mp.getCount(); i++) { final String s = getText(mp.getBodyPart(i), ++depth, preferHtmlContent); if (s != null) { return s; } } } return null; } private static String stripTags(final String text) { if (text == null) { return null; } return text.replaceAll("\\<.*?\\>", ""); } private int attachmentCount; private ESAttachment[] attachments; private Address[] bcc; private Address[] cc; private String contentType; private int flaghashcode; private String[] flags; private String folderFullName; private String folderUri; private Address from; private IndexableHeader[] headers; private Map<String, String> selectedHeaders; private String mailboxType; private final static ObjectMapper mapper = new ObjectMapper(); private String popId; private Date receivedDate; private Date sentDate; private int size; private String subject; private String textContent; private String htmlContent; private Address[] to; private long uid; static { mapper.enable(SerializationConfig.Feature.INDENT_OUTPUT); mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); mapper.configure(SerializationConfig.Feature.WRITE_DATES_AS_TIMESTAMPS, false); mapper.setDateFormat(new SimpleDateFormat("yyyyMMdd'T'HHmmss.SSSZ")); //TODO THREAD safe? } public IndexableMailMessage() { } public String build() throws IOException { return mapper.writeValueAsString(this); } @Override public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } final IndexableMailMessage other = (IndexableMailMessage) obj; if (folderFullName == null) { if (other.folderFullName != null) { return false; } } else if (!folderFullName.equals(other.folderFullName)) { return false; } if (popId == null) { if (other.popId != null) { return false; } } else if (!popId.equals(other.popId)) { return false; } if (uid != other.uid) { return false; } return true; } public int getAttachmentCount() { return attachmentCount; } public ESAttachment[] getAttachments() { return attachments; } public Address[] getBcc() { return bcc; } public Address[] getCc() { return cc; } public String getContentType() { return contentType; } public int getFlaghashcode() { return flaghashcode; } public String[] getFlags() { return flags; } public String getFolderFullName() { return folderFullName; } public String getFolderUri() { return folderUri; } public Address getFrom() { return from; } public IndexableHeader[] getHeaders() { return headers; } @JsonAnyGetter public Map<String, String> getSelectedHeaders() { return selectedHeaders; } public String getMailboxType() { return mailboxType; } public String getPopId() { return popId; } public Date getReceivedDate() { return receivedDate; } public Date getSentDate() { return sentDate; } public int getSize() { return size; } public String getSubject() { return subject; } public String getTextContent() { return textContent; } public String getHtmlContent() { return htmlContent; } public Address[] getTo() { return to; } public long getUid() { return uid; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (folderFullName == null ? 0 : folderFullName.hashCode()); result = prime * result + (popId == null ? 0 : popId.hashCode()); result = prime * result + (int) (uid ^ uid >>> 32); return result; } public void setAttachmentCount(final int attachmentCount) { this.attachmentCount = attachmentCount; } public void setAttachments(final ESAttachment[] attachments) { this.attachments = attachments; } public void setBcc(final Address[] bcc) { this.bcc = bcc; } public void setCc(final Address[] cc) { this.cc = cc; } public void setContentType(final String contentType) { this.contentType = contentType; } public void setFlaghashcode(final int flaghashcode) { this.flaghashcode = flaghashcode; } public void setFlags(final String[] flags) { this.flags = flags; } public void setFolderFullName(final String folderFullName) { this.folderFullName = folderFullName; } public void setFolderUri(final String folderUri) { this.folderUri = folderUri; } public void setFrom(final Address from) { this.from = from; } public void setHeaders(final IndexableHeader[] headers) { this.headers = headers; } public void setSelectedHeaders(Map<String, String> selectedHeaders) { this.selectedHeaders = selectedHeaders; } public void setMailboxType(final String mailboxType) { this.mailboxType = mailboxType; } public void setPopId(final String popId) { this.popId = popId; } public void setReceivedDate(final Date receivedDate) { this.receivedDate = receivedDate; } public void setSentDate(final Date sentDate) { this.sentDate = sentDate; } public void setSize(final int size) { this.size = size; } public void setSubject(final String subject) { this.subject = subject; } public void setTextContent(final String textContent) { this.textContent = textContent; } public void setHtmlContent(final String htmlContent) { this.htmlContent = htmlContent; } public void setTo(final Address[] to) { this.to = to; } public void setUid(final long uid) { this.uid = uid; } @Override public String toString() { return "IndexableMailMessage [uid=" + uid + ", headers=" + headers + ", contentType=" + contentType + ", subject=" + subject + ", size=" + size + ", sentDate=" + sentDate + ", receivedDate=" + receivedDate + ", from=" + from + ", to=" + Arrays.toString(to) + ", cc=" + Arrays.toString(cc) + ", bcc=" + Arrays.toString(bcc) + "]"; } public static class Address { public static Address fromJavaMailAddress(final javax.mail.Address jma) { final Address a = new Address(); final InternetAddress ia = (InternetAddress) jma; a.setEmail(ia.getAddress()); a.setPersonal(ia.getPersonal()); return a; } public static Address[] fromJavaMailAddress(final javax.mail.Address[] jmas) { final Address[] as = new Address[jmas.length]; int i = 0; for (final javax.mail.Address jma : jmas) { final Address a = new Address(); final InternetAddress ia = (InternetAddress) jma; a.setEmail(ia.getAddress()); a.setPersonal(ia.getPersonal()); as[i++] = a; } return as; } private String email; private String personal; public Address() { super(); } @Override public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } final Address other = (Address) obj; if (email == null) { if (other.email != null) { return false; } } else if (!email.equals(other.email)) { return false; } return true; } public String getEmail() { return email; } public String getPersonal() { return personal; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (email == null ? 0 : email.hashCode()); return result; } public void setEmail(final String email) { this.email = email; } public void setPersonal(final String personal) { this.personal = personal; } @Override public String toString() { return "Address [email=" + email + ", personal=" + personal + "]"; } } public static class ESAttachment { private byte[] content; private String contentType; private String fileName; private int size; public ESAttachment() { } public ESAttachment(final String contentType, final byte[] bytes, final String filename) { super(); setContentType(contentType); setContent(bytes); setFilename(filename); } public byte[] getContent() { return content; } public String getContentType() { return contentType; } public String getFilename() { return fileName; } public String getName() { return fileName; } public int getSize() { return size; } public void setContent(final byte[] content) { this.content = content; this.size = content != null ? content.length : 0; } public void setContentType(final String contentType) { this.contentType = contentType; } public void setFilename(final String filename) { this.fileName = filename; } public void setName(final String name) { this.fileName = name; } public void setSize(final int size) { this.size = size; } } public static class IndexableHeader { final String name; final String value; public IndexableHeader(final String name, final String value) { super(); this.name = name; this.value = value; } @Override public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } final IndexableHeader other = (IndexableHeader) obj; if (name == null) { if (other.name != null) { return false; } } else if (!name.equals(other.name)) { return false; } return true; } public String getName() { return name; } public String getValue() { return value; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + (name == null ? 0 : name.hashCode()); return result; } } }