Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package io.hops.erasure_coding; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.BlockMissingException; import java.io.IOException; import java.io.InputStream; import java.util.List; public abstract class StripeReader { public static final Log LOG = LogFactory.getLog(BaseEncodingManager.class); Codec codec; Configuration conf; FileSystem fs; long stripeStartIdx; int bufferSize; public static class LocationPair { private int stripeIdx; private int blockIdxInStripe; public LocationPair(int stripeIdx, int blockIdxInStripe) { this.stripeIdx = stripeIdx; this.blockIdxInStripe = blockIdxInStripe; } int getStripeIdx() { return stripeIdx; } int getBlockIdxInStripe() { return blockIdxInStripe; } } public StripeReader(Configuration conf, Codec codec, FileSystem fs, long stripeStartIdx) { this.codec = codec; this.conf = conf; this.fs = fs; this.stripeStartIdx = stripeStartIdx; this.bufferSize = conf.getInt("io.file.buffer.size", 64 * 1024); } /** * Has next stripe to read? */ public abstract boolean hasNext(); /** * Get the input streams for the next stripe */ public abstract InputStream[] getNextStripeInputs() throws IOException; /** * Builds (codec.stripeLength + codec.parityLength) inputs given some erased * locations. * Outputs: * - the array of input streams @param inputs * - the list of erased locations @param erasedLocations. * - the list of locations that are not read @param locationsToNotRead. */ public InputStream[] buildInputs(FileSystem srcFs, Path srcFile, FileStatus srcStat, FileSystem parityFs, Path parityFile, FileStatus parityStat, int stripeIdx, long offsetInBlock, List<Integer> erasedLocations, List<Integer> locationsToRead, ErasureCode code) throws IOException { InputStream[] inputs = new InputStream[codec.stripeLength + codec.parityLength]; boolean redo = false; do { /* * In the first iteration locationsToRead is empty. * It is populated according to locationsToReadForDecode. * In consecutive iterations (if a stream failed to open) * the list is cleared and re-populated. */ locationsToRead.clear(); locationsToRead.addAll(code.locationsToReadForDecode(erasedLocations)); for (int i = 0; i < inputs.length; i++) { boolean isErased = (erasedLocations.indexOf(i) != -1); boolean shouldRead = (locationsToRead.indexOf(i) != -1); try { InputStream stm = null; if (isErased || !shouldRead) { if (isErased) { LOG.info("Location " + i + " is erased, using zeros"); } else { LOG.info("Location " + i + " need not be read, using zeros"); } stm = new RaidUtils.ZeroInputStream(srcStat.getBlockSize() * ((i < codec.parityLength) ? stripeIdx * codec.parityLength + i : stripeIdx * codec.stripeLength + i - codec.parityLength)); } else { stm = buildOneInput(i, offsetInBlock, srcFs, srcFile, srcStat, parityFs, parityFile, parityStat); } inputs[i] = stm; } catch (IOException e) { if (e instanceof BlockMissingException || e instanceof ChecksumException) { erasedLocations.add(i); redo = true; RaidUtils.closeStreams(inputs); break; } else { throw e; } } } } while (redo); return inputs; } /** * Given a block in the file and specific codec, return the LocationPair * object which contains id of the stripe it belongs to and its * location in the stripe */ public static LocationPair getBlockLocation(Codec codec, int blockIdxInFile) throws IOException { int stripeIdx; int blockIdxInStripe; int blockIdx = blockIdxInFile; stripeIdx = blockIdx / codec.stripeLength; blockIdxInStripe = blockIdx % codec.stripeLength; return new LocationPair(stripeIdx, blockIdxInStripe); } public static LocationPair getParityBlockLocation(Codec codec, final int blockIdxInFile) { int stripeIdx = blockIdxInFile / codec.parityLength; int blockIdxInStripe = blockIdxInFile % codec.parityLength; return new LocationPair(stripeIdx, blockIdxInStripe); } public static StripeReader getStripeReader(Codec codec, Configuration conf, long blockSize, FileSystem fs, long stripeIdx, FileStatus srcStat) throws IOException { return new FileStripeReader(conf, blockSize, codec, fs, stripeIdx, srcStat.getPath(), srcStat.getLen()); } protected abstract InputStream buildOneInput(int locationIndex, long offsetInBlock, FileSystem srcFs, Path srcFile, FileStatus srcStat, FileSystem parityFs, Path parityFile, FileStatus parityStat) throws IOException; protected InputStream getParityFileInput(int locationIndex, Path parityFile, FileSystem parityFs, FileStatus parityStat, long offsetInBlock) throws IOException { // Dealing with a parity file here. int parityBlockIdx = (int) (codec.parityLength * stripeStartIdx + locationIndex); long offset = parityStat.getBlockSize() * parityBlockIdx + offsetInBlock; assert (offset < parityStat.getLen()); LOG.info("Opening " + parityFile + ":" + offset + " for location " + locationIndex); FSDataInputStream s = parityFs.open(parityFile, conf.getInt("io.file.buffer.size", 64 * 1024)); s.seek(offset); return s; } }