Java tutorial
/* The contents of this file are subject to the license and copyright terms * detailed in the license directory at the root of the source tree (also * available online at http://fedora-commons.org/license/). */ package fedora.server.storage.translation; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.Reader; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.text.ParseException; import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import javax.xml.stream.XMLStreamException; import javax.activation.MimeType; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; import org.apache.abdera.Abdera; import org.apache.abdera.ext.thread.ThreadHelper; import org.apache.abdera.i18n.iri.IRI; import org.apache.abdera.model.Category; import org.apache.abdera.model.Content; import org.apache.abdera.model.Document; import org.apache.abdera.model.Entry; import org.apache.abdera.model.Feed; import org.apache.abdera.model.Person; import org.apache.abdera.parser.Parser; import org.apache.abdera.util.MimeTypeHelper; import org.apache.abdera.xpath.XPath; import fedora.common.Constants; import fedora.common.MalformedPIDException; import fedora.common.PID; import fedora.common.xml.format.XMLFormat; import fedora.server.errors.ObjectIntegrityException; import fedora.server.errors.StreamIOException; import fedora.server.errors.ValidationException; import fedora.server.storage.types.Datastream; import fedora.server.storage.types.DatastreamManagedContent; import fedora.server.storage.types.DatastreamReferencedContent; import fedora.server.storage.types.DatastreamXMLMetadata; import fedora.server.storage.types.DigitalObject; import fedora.server.utilities.DateUtility; import fedora.server.validation.ValidationUtility; import fedora.utilities.FileUtils; import fedora.utilities.NormalizedURI; /** * Deserializer for Fedora Objects in Atom format. * * @author Edwin Shin * @since 3.0 * @version $Id$ */ public class AtomDODeserializer implements DODeserializer, Constants { public static final XMLFormat DEFAULT_FORMAT = ATOM1_1; /** Logger for this class. */ private static final Logger LOG = Logger.getLogger(AtomDODeserializer.class); /** The object to deserialize to. */ private DigitalObject m_obj; private String m_encoding; /** The current translation context. */ private int m_transContext; /** The format this deserializer reads. */ private final XMLFormat m_format; private Abdera abdera = Abdera.getInstance(); private Feed m_feed; private XPath m_xpath; private ZipInputStream m_zin; /** * Temporary directory for the unpacked contents of an Atom Zip archive. */ private File m_tempDir; public AtomDODeserializer() { this(DEFAULT_FORMAT); } public AtomDODeserializer(XMLFormat format) { if (format.equals(ATOM1_1) || format.equals(ATOM_ZIP1_1)) { m_format = format; } else { throw new IllegalArgumentException("Not an Atom format: " + format.uri); } } /** * {@inheritDoc} */ public void deserialize(InputStream in, DigitalObject obj, String encoding, int transContext) throws ObjectIntegrityException, StreamIOException, UnsupportedEncodingException { if (m_format.equals(ATOM_ZIP1_1)) { try { m_tempDir = FileUtils.createTempDir("atomzip", null); m_zin = new ZipInputStream(new BufferedInputStream(in)); ZipEntry entry; while ((entry = m_zin.getNextEntry()) != null) { FileUtils.copy(m_zin, new FileOutputStream(new File(m_tempDir, entry.getName()))); } in = new FileInputStream(new File(m_tempDir, "atommanifest.xml")); } catch (FileNotFoundException e) { throw new StreamIOException(e.getMessage(), e); } catch (IOException e) { throw new StreamIOException(e.getMessage(), e); } } Parser parser = abdera.getParser(); Document<Feed> feedDoc = parser.parse(in); m_feed = feedDoc.getRoot(); m_xpath = abdera.getXPath(); m_obj = obj; m_encoding = encoding; m_transContext = transContext; addObjectProperties(); addDatastreams(); DOTranslationUtility.normalizeDatastreams(m_obj, m_transContext, m_encoding); FileUtils.delete(m_tempDir); } /** * {@inheritDoc} */ public DODeserializer getInstance() { return new AtomDODeserializer(m_format); } /** * Set the Fedora Object properties from the Feed metadata. * * @throws ObjectIntegrityException */ private void addObjectProperties() throws ObjectIntegrityException { PID pid; try { pid = new PID(m_feed.getId().toString()); } catch (MalformedPIDException e) { throw new ObjectIntegrityException(e.getMessage(), e); } String label = m_feed.getTitle(); String state = m_xpath.valueOf("/a:feed/a:category[@scheme='" + MODEL.STATE.uri + "']/@term", m_feed); String createDate = m_xpath.valueOf("/a:feed/a:category[@scheme='" + MODEL.CREATED_DATE.uri + "']/@term", m_feed); m_obj.setPid(pid.toString()); try { m_obj.setState(DOTranslationUtility.readStateAttribute(state)); } catch (ParseException e) { throw new ObjectIntegrityException("Could not read object state", e); } m_obj.setLabel(label); m_obj.setOwnerId(getOwnerId()); m_obj.setCreateDate(DateUtility.convertStringToDate(createDate)); m_obj.setLastModDate(m_feed.getUpdated()); setExtProps(); } private void addDatastreams() throws UnsupportedEncodingException, StreamIOException, ObjectIntegrityException { m_feed.sortEntries(new UpdatedIdComparator(true)); List<Entry> entries = m_feed.getEntries(); for (Entry entry : entries) { if (ThreadHelper.getInReplyTo(entry) != null) { addDatastreamVersion(entry); } } } private void addDatastreamVersion(Entry entry) throws UnsupportedEncodingException, StreamIOException, ObjectIntegrityException { IRI ref = ThreadHelper.getInReplyTo(entry).getRef(); Entry parent = m_feed.getEntry(ref.toString()); Datastream ds; String controlGroup = getDSControlGroup(parent); if (controlGroup.equals("X")) { ds = addInlineDatastreamVersion(entry); } else if (controlGroup.equals("M")) { ds = addManagedDatastreamVersion(entry); } else { ds = addExternalReferencedDatastreamVersion(entry); } m_obj.addDatastreamVersion(ds, true); } private Datastream addInlineDatastreamVersion(Entry entry) throws ObjectIntegrityException, StreamIOException { DatastreamXMLMetadata ds = new DatastreamXMLMetadata(); setDSCommonProperties(ds, entry); String dsId = ds.DatastreamID; String dsvId = ds.DSVersionID; ds.DSLocation = m_obj.getPid() + "+" + dsId + "+" + dsvId; if (ds.DSVersionID.equals("AUDIT.0")) { addAuditDatastream(entry); } else { try { if (m_format.equals(ATOM_ZIP1_1)) { ByteArrayOutputStream bout = new ByteArrayOutputStream(); FileUtils.copy(new FileInputStream(getContentSrcAsFile(entry.getContentSrc())), bout); ds.xmlContent = bout.toByteArray(); } else { ds.xmlContent = entry.getContent().getBytes(m_encoding); //IOUtils.toByteArray(entry.getContentStream()); } } catch (UnsupportedEncodingException e) { throw new StreamIOException(e.getMessage(), e); } catch (FileNotFoundException e) { throw new ObjectIntegrityException(e.getMessage(), e); } } if (ds.xmlContent != null) { ds.DSSize = ds.xmlContent.length; } MimeType mimeType = entry.getContentMimeType(); if (mimeType == null) { ds.DSMIME = "text/xml"; } else { ds.DSMIME = mimeType.toString(); } return ds; } private Datastream addExternalReferencedDatastreamVersion(Entry entry) throws ObjectIntegrityException { Datastream ds = new DatastreamReferencedContent(); setDSCommonProperties(ds, entry); ds.DSLocation = entry.getContentSrc().toString(); // Normalize the dsLocation for the deserialization context ds.DSLocation = (DOTranslationUtility.normalizeDSLocationURLs(m_obj.getPid(), ds, m_transContext)).DSLocation; ds.DSLocationType = "URL"; ds.DSMIME = entry.getContentMimeType().toString(); return ds; } private Datastream addManagedDatastreamVersion(Entry entry) throws StreamIOException, ObjectIntegrityException { Datastream ds = new DatastreamManagedContent(); setDSCommonProperties(ds, entry); ds.DSLocationType = "INTERNAL_ID"; ds.DSMIME = getDSMimeType(entry); // Managed Content can take any of the following forms: // 1) inline text (plaintext, html, xml) // 2) inline Base64 // 3) referenced content IRI contentLocation = entry.getContentSrc(); if (contentLocation != null) { // URL FORMAT VALIDATION for dsLocation: // For Managed Content the URL is only checked when we are parsing a // a NEW ingest file because the URL is replaced with an internal identifier // once the repository has sucked in the content for storage. if (m_obj.isNew()) { ValidationUtility.validateURL(contentLocation.toString(), ds.DSControlGrp); } if (m_format.equals(ATOM_ZIP1_1)) { if (!contentLocation.isAbsolute() && !contentLocation.isPathAbsolute()) { File f = getContentSrcAsFile(contentLocation); contentLocation = new IRI(DatastreamManagedContent.TEMP_SCHEME + f.getAbsolutePath()); } } ds.DSLocation = contentLocation.toString(); ds.DSLocation = (DOTranslationUtility.normalizeDSLocationURLs(m_obj.getPid(), ds, m_transContext)).DSLocation; return ds; } try { File temp = File.createTempFile("binary-datastream", null); OutputStream out = new FileOutputStream(temp); if (MimeTypeHelper.isText(ds.DSMIME) || MimeTypeHelper.isXml(ds.DSMIME)) { IOUtils.copy(new StringReader(entry.getContent()), out, m_encoding); } else { IOUtils.copy(entry.getContentStream(), out); } ds.DSLocation = DatastreamManagedContent.TEMP_SCHEME + temp.getAbsolutePath(); } catch (IOException e) { throw new StreamIOException(e.getMessage(), e); } return ds; } private void addAuditDatastream(Entry entry) throws ObjectIntegrityException, StreamIOException { try { Reader auditTrail; if (m_format.equals(ATOM_ZIP1_1)) { File f = getContentSrcAsFile(entry.getContentSrc()); auditTrail = new InputStreamReader(new FileInputStream(f), m_encoding); } else { auditTrail = new StringReader(entry.getContent()); } m_obj.getAuditRecords().addAll(DOTranslationUtility.getAuditRecords(auditTrail)); auditTrail.close(); } catch (XMLStreamException e) { throw new ObjectIntegrityException(e.getMessage(), e); } catch (IOException e) { throw new StreamIOException(e.getMessage(), e); } } private String getOwnerId() { Person owner = m_feed.getAuthor(); if (owner == null) { return ""; } else { return owner.getName(); } } /** * Parses the id to determine a datastreamId. * * @param id * @return */ private String getDatastreamId(Entry entry) { String entryId = entry.getId().toString(); // matches info:fedora/pid/dsid/timestamp Pattern pattern = Pattern.compile("^" + Constants.FEDORA.uri + ".+?/([^/]+)/?.*"); Matcher matcher = pattern.matcher(entryId); if (matcher.find()) { return matcher.group(1); } else { return m_obj.newDatastreamID(); } } private String getDatastreamVersionId(Entry entry) { String dsId = getDatastreamId(entry); String dsvId = entry.getTitle(); // e.g. Match DS1.0 but not DS1 if (dsvId.matches("^" + dsId + ".*\\.[\\w]")) { return dsvId; } else { if (!m_obj.datastreams(dsId).iterator().hasNext()) { return dsId + ".0"; } else { return m_obj.newDatastreamID(dsId); } } } private String getDSControlGroup(Entry entry) throws ObjectIntegrityException { List<Category> controlGroups = entry.getCategories(MODEL.CONTROL_GROUP.uri); // Try to infer the control group if not provided if (controlGroups.isEmpty() || controlGroups.size() > 1) { if (entry.getContentType() != null) { if (entry.getContentType().equals(Content.Type.XML)) { return "X"; } else { // only XML can be inline return "M"; } } if (entry.getContentSrc() != null) { return "M"; } // TODO other cases // link alts, link enclosures else { throw new ObjectIntegrityException("No control group provided by " + m_obj.getPid()); } } else { return controlGroups.get(0).getTerm(); } } private String getDSState(Entry entry) { List<Category> state = entry.getCategories(MODEL.STATE.uri); if (state.isEmpty() || state.size() > 1) { return "A"; } else { return state.get(0).getTerm(); } } /** * Note: AUDIT datastreams always return false, otherwise defaults to true. * * @param entry * @return */ private boolean getDSVersionable(Entry entry) { if (getDatastreamId(entry).equals("AUDIT")) { return false; } List<Category> versionable = entry.getCategories(MODEL.VERSIONABLE.uri); if (versionable.isEmpty() || versionable.size() > 1) { return true; } else { return Boolean.valueOf(versionable.get(0).getTerm()); } } private String[] getDSAltIds(Entry entry) { List<Category> altIds = entry.getCategories(MODEL.ALT_IDS.uri); if (altIds.isEmpty()) { return new String[0]; } else { return altIds.get(0).getTerm().split(" "); // TODO we could handle size > 1 } } private String getDSFormatURI(Entry entry) { List<Category> formatURI = entry.getCategories(MODEL.FORMAT_URI.uri); if (formatURI.isEmpty() || formatURI.size() > 1) { return null; } else { return formatURI.get(0).getTerm(); } } private String getDSLabel(Entry entry) { List<Category> label = entry.getCategories(MODEL.LABEL.uri); if (label.isEmpty()) { return ""; } return label.get(0).getTerm(); } private String getDSMimeType(Entry entry) { String dsMimeType = "application/unknown"; MimeType mimeType = entry.getContentMimeType(); if (mimeType == null) { Content.Type type = entry.getContentType(); if (type != null) { if (type == Content.Type.HTML) { dsMimeType = "text/html"; } else if (type == Content.Type.TEXT) { dsMimeType = "text/plain"; } else if (type == Content.Type.XHTML) { dsMimeType = "application/xhtml+xml"; } else if (type == Content.Type.XML) { dsMimeType = "text/xml"; } } } else { dsMimeType = mimeType.toString(); } return dsMimeType; } private String getDSChecksumType(Entry entry) { List<Category> digestType = entry.getCategories(MODEL.DIGEST_TYPE.uri); if (digestType.isEmpty()) { return Datastream.CHECKSUMTYPE_DISABLED; } else { return digestType.get(0).getTerm(); } } private String getDSChecksum(Entry entry) { List<Category> digest = entry.getCategories(MODEL.DIGEST.uri); if (digest.isEmpty()) { return Datastream.CHECKSUM_NONE; } else { return digest.get(0).getTerm(); } } private void setDSCommonProperties(Datastream dsVersion, Entry entry) throws ObjectIntegrityException { IRI ref = ThreadHelper.getInReplyTo(entry).getRef(); Entry parent = m_feed.getEntry(ref.toString()); dsVersion.DatastreamID = getDatastreamId(parent); dsVersion.DSControlGrp = getDSControlGroup(parent); dsVersion.DSState = getDSState(parent); dsVersion.DSVersionable = getDSVersionable(parent); setDatastreamVersionProperties(dsVersion, entry); } private void setDatastreamVersionProperties(Datastream ds, Entry entry) throws ValidationException { ds.DatastreamAltIDs = getDSAltIds(entry); ds.DSCreateDT = entry.getUpdated(); ds.DSFormatURI = getDSFormatURI(entry); ds.DSLabel = getDSLabel(entry); ds.DSVersionID = getDatastreamVersionId(entry); ds.DSChecksumType = getDSChecksumType(entry); String checksum = getDSChecksum(entry); if (m_obj.isNew()) { if (LOG.isDebugEnabled()) { LOG.debug("New Object: checking supplied checksum"); } if (checksum != null && !checksum.equals("") && !checksum.equals(Datastream.CHECKSUM_NONE)) { String tmpChecksum = ds.getChecksum(); if (LOG.isDebugEnabled()) { LOG.debug("checksum = " + tmpChecksum); } if (!checksum.equals(tmpChecksum)) { throw new ValidationException("Checksum Mismatch: " + tmpChecksum); } } ds.DSChecksumType = ds.getChecksumType(); } else { ds.DSChecksum = checksum; } } private void setExtProps() { List<Category> epCategories = m_feed.getCategories(MODEL.EXT_PROPERTY.uri); for (Category epCategory : epCategories) { m_obj.setExtProperty(epCategory.getTerm(), epCategory.getLabel()); } } /** * Returns the an Entry's contentSrc as a File relative to {@link #m_tempDir}. * * @param contentSrc * @return the contentSrc as a File relative to m_tempDir. * @throws ObjectIntegrityException */ protected File getContentSrcAsFile(IRI contentSrc) throws ObjectIntegrityException { if (contentSrc.isAbsolute() || contentSrc.isPathAbsolute()) { throw new ObjectIntegrityException("contentSrc must not be absolute"); } try { // Normalize the IRI to resolve percent-encoding and // backtracking (e.g. "../") NormalizedURI nUri = new NormalizedURI(m_tempDir.toURI().toString() + contentSrc.toString()); nUri.normalize(); File f = new File(nUri.toURI()); if (f.getParentFile().equals(m_tempDir)) { return f; } else { throw new ObjectIntegrityException(contentSrc.toString() + " is not a valid path."); } } catch (URISyntaxException e) { throw new ObjectIntegrityException(e.getMessage(), e); } } private static class UpdatedIdComparator implements Comparator<Entry> { private boolean ascending = true; UpdatedIdComparator(boolean ascending) { this.ascending = ascending; } public int compare(Entry o1, Entry o2) { Date d1 = o1.getUpdated(); Date d2 = o2.getUpdated(); String id1 = o1.getId().toString(); String id2 = o2.getId().toString(); int r = d1.compareTo(d2); if (d1.equals(d2)) { r = id1.compareTo(id2); } return (ascending) ? r : -r; } } }