Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package io.milton.zsync; import io.milton.common.RangeUtils; import io.milton.common.StreamUtils; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.nio.channels.FileChannel; import java.util.Enumeration; import java.util.List; import java.util.NoSuchElementException; import org.apache.commons.lang.StringUtils; import io.milton.http.Range; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * An object that performs the server side operations needed to assemble the * file from a ZSync PUT. * <p/> * * These operations consist of copying byte ranges into the new file. The * {@link #moveBlocks} method copies ranges from the previous file according to * a list of RelocateRanges, while the {@link #sendRanges} method copies the new * data received in the upload. Both of these methods are overloaded with * versions that accept File rather than byte[] arguments for dealing with large * files that should not be loaded into memory all at once.<p/> * * To assemble the file from a ZSync upload, the server should construct an * UploadReader, passing to the constructor the file to be updated and an * InputStream containing the upload data. It should then invoke the * {@link #assemble()} method, which will return a temporary file that was * created. * * @author Nick * */ public class UploadReader { private static final Logger log = LoggerFactory.getLogger(UploadReader.class); /** * Copies blocks of data from the in array to the out array. * * @param in The byte array containing the server's file being replaced * @param rlist The List of RelocateRanges received from the upload * @param blockSize The block size used in rlist * @param out The byte array of the file being assembled */ public static void moveBlocks(byte[] in, List<RelocateRange> rlist, int blockSize, byte[] out) { for (RelocateRange reloc : rlist) { int startBlock = (int) reloc.getBlockRange().getStart().longValue(); int finishBlock = (int) reloc.getBlockRange().getFinish().longValue(); int startByte = startBlock * blockSize; int newOffset = (int) reloc.getOffset(); int numBytes = (finishBlock - startBlock) * blockSize; System.arraycopy(in, startByte, out, newOffset, numBytes); } } /** * Copies blocks of data from the input File to the output File. For each * RelocateRange A-B/C in relocRanges, the block starting at A and ending at * B-1 is copied from inFile and written to byte C of outFile. * * @param inFile The server's File being replaced * @param relocRanges The Enumeration of RelocateRanges parsed from the * Upload's relocStream * @param blocksize The block size used in relocRanges * @param outFile The File being assembled * @throws IOException */ public static void moveBlocks(File inFile, Enumeration<RelocateRange> relocRanges, int blocksize, File outFile) throws IOException { /* * Because transferFrom can supposedly throw Exceptions when copying large Files, * this method invokes moveRange to copy incrementally */ /*The FileChannels should be obtained from a RandomAccessFile rather than a *Stream, or the position() method will not work correctly */ FileChannel rc = null; FileChannel wc = null; try { rc = new RandomAccessFile(inFile, "r").getChannel(); wc = new RandomAccessFile(outFile, "rw").getChannel(); while (relocRanges.hasMoreElements()) { moveRange(rc, relocRanges.nextElement(), blocksize, wc); } } finally { Util.close(rc); Util.close(wc); } } /** * Copies a Range of blocks from rc into a new offset of wc * * @param rc A FileChannel for the input File * @param reloc The RelocateRange specifying the Range to be copied and its * new offset * @param blockSize The block size used by reloc * @param wc The FileChannel for the output File * @throws IOException */ private static void moveRange(FileChannel rc, RelocateRange reloc, int blockSize, FileChannel wc) throws IOException { long MAX_BUFFER = 16384; long startBlock = reloc.getBlockRange().getStart(); long finishBlock = reloc.getBlockRange().getFinish(); long bytesLeft = (finishBlock - startBlock) * blockSize; //bytes left to copy long readAtOnce = 0; //number of bytes to attempt to read long bytesRead = 0; //number of bytes actually read long currOffset = reloc.getOffset(); //current write position if (finishBlock * blockSize > rc.size() || startBlock < 0) { throw new RuntimeException("Invalid RelocateRange: Source file does not contain blocks " + reloc.getBlockRange().getRange()); } rc.position(startBlock * blockSize); while (bytesLeft > 0) { readAtOnce = Math.min(bytesLeft, MAX_BUFFER); /*Because transferFrom does not update the write channel's position, * it needs to be set manually */ bytesRead = wc.transferFrom(rc, currOffset, readAtOnce); bytesLeft -= bytesRead; currOffset += bytesRead; } } /** * Copies bytes from the in array into Ranges of the out array. The in array * is expected to contain the queued bytes in the same order as the ranges * List. * * @param in An array containing the queued bytes corresponding to the * ranges List * @param ranges The List of target Ranges * @param out The byte array for the file being assembled */ public static void sendRanges(byte[] in, List<Range> ranges, byte[] out) { int pos = 0; for (Range r : ranges) { int length = (int) (r.getFinish() - r.getStart()); System.arraycopy(in, pos, out, r.getStart().intValue(), length); pos += length; } } /** * Inserts the data from each DataRange into the output File, at the * appropriate offset * * @param byteRanges The Enumeration of Range/InputStream pairs parsed from * the Upload's dataStream * @param outFile The output File being assembled * @throws IOException */ public static void sendRanges(Enumeration<ByteRange> byteRanges, File outFile) throws IOException { int BUFFER_SIZE = 16384; byte[] buffer = new byte[BUFFER_SIZE]; RandomAccessFile randAccess = null; try { randAccess = new RandomAccessFile(outFile, "rw"); while (byteRanges.hasMoreElements()) { ByteRange byteRange = byteRanges.nextElement(); Range range = byteRange.getRange(); InputStream data = byteRange.getDataQueue(); sendBytes(data, range, buffer, randAccess); } } finally { Util.close(randAccess); } } /** * Reads a number of bytes from the InputStream equal to the size of the * specified Range and writes them into that Range of the RandomAccessFile. * * @param dataIn The InputStream containing the data to be copied * @param range The target location in the RandomAccessFile * @param buffer A byte array used to transfer data from dataIn to fileOut * @param fileOut A RandomAccessFile for the File being assembled * @throws IOException */ private static void sendBytes(InputStream dataIn, Range range, byte[] buffer, RandomAccessFile fileOut) throws IOException { long bytesLeft = (range.getFinish() - range.getStart()); int bytesRead = 0; int readAtOnce = 0; fileOut.seek(range.getStart()); while (bytesLeft > 0) { readAtOnce = (int) Math.min(buffer.length, bytesLeft); bytesRead = dataIn.read(buffer, 0, readAtOnce); fileOut.write(buffer, 0, bytesRead); bytesLeft -= bytesRead; if (bytesLeft > 0 && bytesRead < 0) { throw new RuntimeException("Unable to copy byte Range: " + range.getRange() + ". End of InputStream reached with " + bytesLeft + " bytes left."); } } } /** * Copies the contents of the source file to the destination file and sets * the destination file's length. * * @param inFile The source file * @param outFile The destination file * @param length The desired length of the destination file * @throws IOException */ private static void copyFile(File inFile, File outFile, long length) throws IOException { InputStream fIn = null; OutputStream fOut = null; RandomAccessFile randAccess = null; try { fIn = new FileInputStream(inFile); fOut = new FileOutputStream(outFile); RangeUtils.sendBytes(fIn, fOut, inFile.length()); } finally { StreamUtils.close(fIn); StreamUtils.close(fOut); } try { randAccess = new RandomAccessFile(outFile, "rw"); randAccess.setLength(length); } finally { Util.close(randAccess); } } private final File serverCopy; private final File uploadedCopy; private final Upload uploadData; /** * Constructor that parses the InputStream into an Upload object and * initializes a temporary file that will contain the assembled upload * * @param serverFile The server file to be updated * @param uploadIn A stream containing the ZSync PUT data * @throws IOException */ public UploadReader(File serverFile, InputStream uploadIn) throws IOException { this.serverCopy = serverFile; this.uploadData = Upload.parse(uploadIn); this.uploadedCopy = File.createTempFile("zsync-upload", "newFile"); } /** * Invokes the methods to put together the uploaded file. * * @return The assembled File * @throws IOException */ public File assemble() throws IOException { if (uploadData.getBlocksize() <= 0) { throw new RuntimeException("Invalid blocksize specified: " + uploadData.getBlocksize()); } if (uploadData.getFilelength() <= 0) { throw new RuntimeException("Invalid file length specified: " + uploadData.getFilelength()); } if (StringUtils.isBlank(uploadData.getSha1())) { throw new RuntimeException("No SHA1 checksum provided."); } InputStream relocIn = null; InputStream dataIn = null; try { relocIn = uploadData.getRelocStream(); dataIn = uploadData.getDataStream(); Enumeration<RelocateRange> relocEnum = new RelocateParser(relocIn); Enumeration<ByteRange> dataEnum = new ByteRangeParser(dataIn); copyFile(serverCopy, uploadedCopy, uploadData.getFilelength()); moveBlocks(serverCopy, relocEnum, (int) uploadData.getBlocksize(), uploadedCopy); sendRanges(dataEnum, uploadedCopy); } finally { StreamUtils.close(relocIn); StreamUtils.close(dataIn); } return uploadedCopy; } /** * Returns the expected SHA1 checksum String received in the upload * * @return A SHA1 checksum */ public String getChecksum() { return uploadData.getSha1(); } /** * An object that wraps the relocate stream of Upload ( * {@link Upload#getRelocStream} )in an Enumeration of RelocateRanges. The * relocate stream is expected to contain a comma separated list of * RelocateRanges, e.g.<p/> * * 10-20/123, 100-200/789 * <p/> * * A few whitespaces at the beginning or end of the list are ignored, as are * those surrounding the commas. * * @author Nick * */ private static class RelocateParser implements Enumeration<RelocateRange> { private InputStream relocIn; private String nextToken; private byte[] COMMA = new byte[1]; /** * Constructs the Enumeration of RelocateRanges from an InputStream * * @param relocIn An InputStream obtained from * {@link Upload#getRelocStream()} */ public RelocateParser(InputStream relocIn) { try { this.relocIn = relocIn; this.COMMA[0] = ",".getBytes(Upload.CHARSET)[0]; this.nextToken = Upload.readToken(relocIn, COMMA, 64); } catch (Exception ex) { throw new RuntimeException(ex); } } @Override public boolean hasMoreElements() { return !StringUtils.isBlank(nextToken); } @Override public RelocateRange nextElement() { if (!this.hasMoreElements()) { throw new NoSuchElementException("No more RelocateRanges"); } try { RelocateRange reloc = RelocateRange.parse(nextToken); nextToken = Upload.readToken(relocIn, COMMA, 64); return reloc; } catch (Exception ex) { throw new RuntimeException(ex); } } } /** * An object that wraps the data stream portion of an Upload in an * Enumeration of ByteRanges. </p> * * In order for the parsing to work, the proper number of bytes must be read * from each ByteRange returned by {@link #nextElement()} prior to the next * invocation of {@link #hasMoreElements()}. * * @author Nick * */ private static class ByteRangeParser implements Enumeration<ByteRange> { /*The dataStream portion of an Upload*/ private final InputStream dataQueue; /*The Range of the next ByteRange. A null value means that the next Range has not *been loaded or that the end of the data section has been reached. */ private Range nextRange; /*Whether an attempt has been made to read the next Range KV pair*/ private boolean rangeloaded; private final byte[] COLON = { ":".getBytes(Upload.CHARSET)[0] }; /** * Constructs the Enumeration from the specified InputStream * * @param in The InputStream obtained from * {@link Upload#getDataStream()} * @throws UnsupportedEncodingException */ public ByteRangeParser(InputStream in) throws UnsupportedEncodingException { this.dataQueue = in; this.rangeloaded = false; } @Override public boolean hasMoreElements() { /* * If rangeloaded == false, attempt to read the next Range KV pair and set rangeloaded = true. * If rangeloaded == true and nextRange == null, there are no further ByteRanges. * */ try { if (rangeloaded) { return nextRange != null; } String nextKey = Upload.readToken(dataQueue, COLON, 64).trim(); if (StringUtils.isBlank(nextKey)) { nextRange = null; } else if (!nextKey.equalsIgnoreCase(Upload.RANGE)) { throw new RuntimeException("Invalid key. Expected: " + Upload.RANGE + "\tActual: " + nextKey); } else { nextRange = Range.parse(Upload.readValue(dataQueue, 64).trim()); } rangeloaded = true; return nextRange != null; } catch (Exception ex) { throw new RuntimeException(ex); } } @Override public ByteRange nextElement() { if (!hasMoreElements()) { throw new NoSuchElementException("No more ByteRanges"); } this.rangeloaded = false; //Reset rangeloaded return new ByteRange(nextRange, dataQueue); } } }