Java tutorial
/*********************************************************************************************************************** * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu) * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. **********************************************************************************************************************/ package eu.stratosphere.nephele.services.iomanager; import java.io.DataInput; import java.io.DataOutput; import java.io.EOFException; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Random; import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.After; import org.junit.Before; import org.junit.Test; import eu.stratosphere.core.io.IOReadableWritable; import eu.stratosphere.core.memory.MemorySegment; import eu.stratosphere.nephele.services.memorymanager.DefaultMemoryManagerTest; import eu.stratosphere.nephele.services.memorymanager.spi.DefaultMemoryManager; import eu.stratosphere.nephele.template.AbstractInvokable; /** * Integration test case for the I/O manager. * * */ public class IOManagerITCase { private static final Log LOG = LogFactory.getLog(IOManagerITCase.class); private static final long SEED = 649180756312423613L; private static final int NUMBER_OF_SEGMENTS = 10; // 10 private static final int SEGMENT_SIZE = 1024 * 1024; // 1M private final int NUM_CHANNELS = 29; private final int NUMBERS_TO_BE_WRITTEN = NUM_CHANNELS * 1000000; private IOManager ioManager; private DefaultMemoryManager memoryManager; @Before public void beforeTest() { memoryManager = new DefaultMemoryManager(NUMBER_OF_SEGMENTS * SEGMENT_SIZE); ioManager = new IOManager(); } @After public void afterTest() throws Exception { ioManager.shutdown(); Assert.assertTrue("IO Manager has not properly shut down.", ioManager.isProperlyShutDown()); Assert.assertTrue("Not all memory was returned to the memory manager in the test.", memoryManager.verifyEmpty()); memoryManager.shutdown(); memoryManager = null; } // ------------------------------------------------------------------------ /** * This test instantiates multiple channels and writes to them in parallel and re-reads the data in * parallel. It is designed to check the ability of the IO manager to correctly handle multiple threads. */ @Test public void parallelChannelsTest() throws Exception { LOG.info("Starting parallel channels test."); final Random rnd = new Random(SEED); final AbstractInvokable memOwner = new DefaultMemoryManagerTest.DummyInvokable(); Channel.ID[] ids = new Channel.ID[NUM_CHANNELS]; BlockChannelWriter[] writers = new BlockChannelWriter[NUM_CHANNELS]; BlockChannelReader[] readers = new BlockChannelReader[NUM_CHANNELS]; ChannelWriterOutputView[] outs = new ChannelWriterOutputView[NUM_CHANNELS]; ChannelReaderInputView[] ins = new ChannelReaderInputView[NUM_CHANNELS]; int[] writingCounters = new int[NUM_CHANNELS]; int[] readingCounters = new int[NUM_CHANNELS]; // instantiate the channels and writers for (int i = 0; i < NUM_CHANNELS; i++) { ids[i] = this.ioManager.createChannel(); writers[i] = this.ioManager.createBlockChannelWriter(ids[i]); List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2); outs[i] = new ChannelWriterOutputView(writers[i], memSegs, this.memoryManager.getPageSize()); } Value val = new Value(); // write a lot of values unevenly distributed over the channels int nextLogCount = 0; float nextLogFraction = 0.0f; LOG.info("Writing to channels..."); for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) { if (i == nextLogCount) { LOG.info("... " + (int) (nextLogFraction * 100) + "% done."); nextLogFraction += 0.05; nextLogCount = (int) (nextLogFraction * NUMBERS_TO_BE_WRITTEN); } int channel = skewedSample(rnd, NUM_CHANNELS - 1); val.value = String.valueOf(writingCounters[channel]++); val.write(outs[channel]); } LOG.info("Writing done, flushing contents..."); // close all writers for (int i = 0; i < NUM_CHANNELS; i++) { this.memoryManager.release(outs[i].close()); } outs = null; writers = null; // instantiate the readers for sequential read LOG.info("Reading channels sequentially..."); for (int i = 0; i < NUM_CHANNELS; i++) { List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2); LOG.info("Reading channel " + (i + 1) + "/" + NUM_CHANNELS + '.'); final BlockChannelReader reader = this.ioManager.createBlockChannelReader(ids[i]); final ChannelReaderInputView in = new ChannelReaderInputView(reader, memSegs, false); int nextVal = 0; try { while (true) { val.read(in); int intValue = 0; try { intValue = Integer.parseInt(val.value); } catch (NumberFormatException nfex) { Assert.fail("Invalid value read from reader. Valid decimal number expected."); } Assert.assertEquals("Written and read values do not match during sequential read.", nextVal, intValue); nextVal++; } } catch (EOFException eofex) { // expected } Assert.assertEquals("NUmber of written numbers differs from number of read numbers.", writingCounters[i], nextVal); this.memoryManager.release(in.close()); } LOG.info("Sequential reading done."); // instantiate the readers LOG.info("Reading channels randomly..."); for (int i = 0; i < NUM_CHANNELS; i++) { List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(NUMBER_OF_SEGMENTS - 2) + 2); readers[i] = this.ioManager.createBlockChannelReader(ids[i]); ins[i] = new ChannelReaderInputView(readers[i], memSegs, false); } nextLogCount = 0; nextLogFraction = 0.0f; // read a lot of values in a mixed order from the channels for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) { if (i == nextLogCount) { LOG.info("... " + (int) (nextLogFraction * 100) + "% done."); nextLogFraction += 0.05; nextLogCount = (int) (nextLogFraction * NUMBERS_TO_BE_WRITTEN); } while (true) { final int channel = skewedSample(rnd, NUM_CHANNELS - 1); if (ins[channel] != null) { try { val.read(ins[channel]); int intValue; try { intValue = Integer.parseInt(val.value); } catch (NumberFormatException nfex) { Assert.fail("Invalid value read from reader. Valid decimal number expected."); return; } Assert.assertEquals("Written and read values do not match.", readingCounters[channel]++, intValue); break; } catch (EOFException eofex) { this.memoryManager.release(ins[channel].close()); ins[channel] = null; } } } } LOG.info("Random reading done."); // close all readers for (int i = 0; i < NUM_CHANNELS; i++) { if (ins[i] != null) { this.memoryManager.release(ins[i].close()); } readers[i].closeAndDelete(); } ins = null; readers = null; // check that files are deleted for (int i = 0; i < NUM_CHANNELS; i++) { File f = new File(ids[i].getPath()); Assert.assertFalse("Channel file has not been deleted.", f.exists()); } } private static final int skewedSample(Random rnd, int max) { double uniform = rnd.nextDouble(); double var = Math.pow(uniform, 8.0); double pareto = 0.2 / var; int val = (int) pareto; return val > max ? val % max : val; } // ------------------------------------------------------------------------ protected static class Value implements IOReadableWritable { String value; public Value() { } public Value(String val) { this.value = val; } @Override public void read(DataInput in) throws IOException { value = in.readUTF(); } @Override public void write(DataOutput out) throws IOException { out.writeUTF(this.value); } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((value == null) ? 0 : value.hashCode()); return result; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } Value other = (Value) obj; if (value == null) { if (other.value != null) { return false; } } else if (!value.equals(other.value)) { return false; } return true; } } }