Java tutorial
/** * Copyright 2011-2017 Asakusa Framework Team. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.asakusafw.runtime.directio.hadoop; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.text.MessageFormat; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.CompressionType; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.util.ReflectionUtils; import com.asakusafw.runtime.directio.Counter; import com.asakusafw.runtime.io.ModelInput; import com.asakusafw.runtime.io.ModelOutput; /** * Data model format of {@link InputStream} / {@link OutputStream} . * This implementation class must have a public constructor without any parameters. * @param <K> the type of raw sequence file key * @param <V> the type of raw sequence file value * @param <T> the type of target data model * @since 0.2.6 * @version 0.4.0 */ public abstract class SequenceFileFormat<K, V, T> extends HadoopFileFormat<T> { static final Log LOG = LogFactory.getLog(SequenceFileFormat.class); static final String KEY_COMPRESSION_CODEC = "com.asakusafw.output.sequencefile.compression.codec"; //$NON-NLS-1$ static final String VALUE_COMPRESSION_AUTO = "auto"; //$NON-NLS-1$ @Override public long getMinimumFragmentSize() throws IOException, InterruptedException { return SequenceFile.SYNC_INTERVAL; } /** * Returns a key object. * @return a key object */ protected abstract K createKeyObject(); /** * Returns a value object. * @return a value object */ protected abstract V createValueObject(); /** * Copy key and value into the target data model. * @param key the source key object * @param value the source value object * @param model the target data model * @throws IOException if failed to copy */ protected abstract void copyToModel(K key, V value, T model) throws IOException; /** * Copy the data model into the key and value. * @param model the source data model * @param key the target key object * @param value the target value object * @throws IOException if failed to copy */ protected abstract void copyFromModel(T model, K key, V value) throws IOException; @Override public ModelInput<T> createInput(Class<? extends T> dataType, FileSystem fileSystem, Path path, long offset, long fragmentSize, Counter counter) throws IOException, InterruptedException { long end = offset + fragmentSize; K keyBuffer = createKeyObject(); V valueBuffer = createValueObject(); SequenceFile.Reader reader; try { reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(fileSystem.makeQualified(path))); } catch (EOFException e) { FileStatus status = fileSystem.getFileStatus(path); if (status.getLen() == 0L) { LOG.warn(MessageFormat.format("Target sequence file is empty: {0}", path)); return new ModelInput<T>() { @Override public boolean readTo(T model) throws IOException { return false; } @Override public void close() throws IOException { return; } }; } throw e; } boolean succeed = false; try { if (offset > reader.getPosition()) { reader.sync(offset); } ModelInput<T> result = new ModelInput<T>() { private boolean next = reader.getPosition() < end; private long lastPosition = reader.getPosition(); @Override public boolean readTo(T model) throws IOException { if (next == false) { return false; } long current = reader.getPosition(); @SuppressWarnings("unchecked") K key = (K) reader.next(keyBuffer); if (key == null || (current >= end && reader.syncSeen())) { next = false; return false; } else { reader.getCurrentValue(valueBuffer); SequenceFileFormat.this.copyToModel(keyBuffer, valueBuffer, model); long nextPosition = reader.getPosition(); counter.add(nextPosition - lastPosition); lastPosition = nextPosition; return true; } } @Override public void close() throws IOException { reader.close(); } }; succeed = true; return result; } finally { if (succeed == false) { reader.close(); } } } @Override public ModelOutput<T> createOutput(Class<? extends T> dataType, FileSystem fileSystem, Path path, Counter counter) throws IOException, InterruptedException { K keyBuffer = createKeyObject(); V valueBuffer = createValueObject(); CompressionCodec codec = getCompressionCodec(path); if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format("Creating sequence file (path={0}, type={1}, codec={2})", //$NON-NLS-1$ path, dataType.getName(), codec)); } configure(codec); SequenceFile.Writer writer = SequenceFile.createWriter(getConf(), SequenceFile.Writer.file(fileSystem.makeQualified(path)), SequenceFile.Writer.keyClass(keyBuffer.getClass()), SequenceFile.Writer.valueClass(valueBuffer.getClass()), SequenceFile.Writer .compression(codec == null ? CompressionType.NONE : CompressionType.BLOCK, codec)); boolean succeed = false; try { ModelOutput<T> output = new ModelOutput<T>() { private long lastPosition = 0; @Override public void write(T model) throws IOException { copyFromModel(model, keyBuffer, valueBuffer); writer.append(keyBuffer, valueBuffer); long nextPosition = writer.getLength(); counter.add(nextPosition - lastPosition); lastPosition = nextPosition; } @Override public void close() throws IOException { writer.close(); } }; succeed = true; return output; } finally { if (succeed == false) { writer.close(); } } } private void configure(Object object) { if (object instanceof Configurable) { Configurable configurable = (Configurable) object; if (configurable.getConf() == null) { configurable.setConf(getConf()); } } } /** * Returns a compression codec for output sequence files. * Clients can override this method in subclasses, and return the suitable {@link CompressionCodec} object. * @param path target path * @return a compression codec used to output, or {@code null} if output will not be compressed * @throws IOException if failed to create a compression codec * @throws InterruptedException if interrupted */ public CompressionCodec getCompressionCodec(Path path) throws IOException, InterruptedException { String codecClassName = getConf().get(KEY_COMPRESSION_CODEC); if (codecClassName != null && codecClassName.isEmpty() == false) { try { Class<?> codecClass = getConf().getClassByName(codecClassName); return ReflectionUtils.newInstance(codecClass.asSubclass(CompressionCodec.class), getConf()); } catch (Exception e) { LOG.warn(MessageFormat.format("Failed to load compression codec ({0}={1})", KEY_COMPRESSION_CODEC, codecClassName), e); return null; } } return null; } }