Java tutorial
package com.splout.db.dnode; /* * #%L * Splout SQL Server * %% * Copyright (C) 2012 Datasalt Systems S.L. * %% * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * #L% */ import com.google.common.io.Files; import com.splout.db.common.SploutConfiguration; import com.splout.db.dnode.Fetcher.Throttler; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Test; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.charset.Charset; import static org.junit.Assert.*; public class TestFetcher { @Test public void testThrottling() throws InterruptedException { double bytesPerSec = 1000; Throttler throttler = new Throttler(bytesPerSec); long startTime = System.currentTimeMillis(); int bytesConsumed = 0; for (int i = 0; i < 10; i++) { int bytes = (int) (Math.random() * 1000); bytesConsumed += bytes; throttler.incrementAndThrottle(bytes); } long endTime = System.currentTimeMillis(); double secs = (endTime - startTime) / (double) 1000; double avgBytesPerSec = bytesConsumed / secs; assertEquals(bytesPerSec, avgBytesPerSec, 5.0); // + - 5 } @Test public void testHdfsFetching() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); Fetcher fetcher = new Fetcher(testConfig); Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write("This is what happens when you don't know what to write".getBytes()); oS.close(); File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals("This is what happens when you don't know what to write", Files.toString(file, Charset.defaultCharset())); fS.delete(path, true); FileUtils.deleteDirectory(f); } @Test public void testHdfsFetchingInterrupted() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); final FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); final Fetcher fetcher = new Fetcher(testConfig); final Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write("This is what happens when you don't know what to write".getBytes()); oS.close(); Thread t = new Thread() { @Override public void run() { try { try { File f = fetcher .fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); } catch (IOException e) { e.printStackTrace(); } catch (URISyntaxException e) { e.printStackTrace(); } fail("An InterruptedException was expected."); } catch (InterruptedException e) { // Everything good. } } }; // We interrupt the thread before starting so we are sure that the interruption check // will be seen even if the file to copy is very small. t.interrupt(); t.start(); } @Test public void testHdfsFetchingAndThrottling() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); FileSystem fS = FileSystem.getLocal(conf); SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); testConfig.setProperty(FetcherProperties.DOWNLOAD_BUFFER, 4); testConfig.setProperty(FetcherProperties.BYTES_PER_SEC_THROTTLE, 8); Fetcher fetcher = new Fetcher(testConfig); final String str = "This is what happens when you don't know what to write"; Path path = new Path("tmp-" + TestFetcher.class.getName()); OutputStream oS = fS.create(path); oS.write(str.getBytes()); oS.close(); long startTime = System.currentTimeMillis(); File f = fetcher.fetch(new Path(fS.getWorkingDirectory(), path.getName()).toUri().toString()); long endTime = System.currentTimeMillis(); double bytesPerSec = (str.getBytes().length / (double) (endTime - startTime)) * 1000; assertEquals(8, bytesPerSec, 0.5); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals(str, Files.toString(file, Charset.defaultCharset())); fS.delete(path, true); FileUtils.deleteDirectory(f); } @Test public void testFileFetching() throws IOException, URISyntaxException, InterruptedException { SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); Fetcher fetcher = new Fetcher(testConfig); File file = new File("tmp-" + TestFetcher.class.getName()); Files.write("This is what happens when you don't know what to write".getBytes(), file); File f = fetcher.fetch(file.getAbsoluteFile().toURI().toString()); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file2 = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals("This is what happens when you don't know what to write", Files.toString(file2, Charset.defaultCharset())); file.delete(); FileUtils.deleteDirectory(f); } @Test public void testFileFetchingAndThrottling() throws IOException, URISyntaxException, InterruptedException { SploutConfiguration testConfig = SploutConfiguration.getTestConfig(); testConfig.setProperty(FetcherProperties.TEMP_DIR, "tmp-dir-" + TestFetcher.class.getName()); testConfig.setProperty(FetcherProperties.DOWNLOAD_BUFFER, 4); testConfig.setProperty(FetcherProperties.BYTES_PER_SEC_THROTTLE, 8); Fetcher fetcher = new Fetcher(testConfig); final String str = "This is what happens when you don't know what to write"; File file = new File("tmp-" + TestFetcher.class.getName()); Files.write(str.getBytes(), file); long startTime = System.currentTimeMillis(); File f = fetcher.fetch(file.getAbsoluteFile().toURI().toString()); long endTime = System.currentTimeMillis(); double bytesPerSec = (str.getBytes().length / (double) (endTime - startTime)) * 1000; assertEquals(8, bytesPerSec, 0.5); assertTrue(f.exists()); assertTrue(f.isDirectory()); File file2 = new File(f, "tmp-" + TestFetcher.class.getName()); assertTrue(file.exists()); assertEquals(str, Files.toString(file2, Charset.defaultCharset())); file.delete(); FileUtils.deleteDirectory(f); } }