Java tutorial
/** * Copyright 2011-2015 Asakusa Framework Team. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.asakusafw.cleaner.main; import static org.junit.Assert.*; import java.io.File; import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.Properties; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import com.asakusafw.cleaner.common.ConfigurationLoader; import com.asakusafw.cleaner.testutil.UnitTestUtil; /** * HDFSCleaner? * * * @author yuta.shirai * */ public class HDFSCleanerTest { // private static final String propFile = "clean-hdfs-conf.properties"; private static final String propFile1 = "clean-hdfs-conf1.properties"; private static final String propFile2 = "clean-hdfs-conf2.properties"; private static final String propFile3 = "clean-hdfs-conf3.properties"; private static final String propFile4 = "clean-hdfs-conf4.properties"; private static final String propFile5 = "clean-hdfs-conf5.properties"; private static final String propFile6 = "clean-hdfs-conf6.properties"; // private static final File cleanDir01 = new File("target/asakusa-cleaner/HDFSCleaner01"); // 1 private File tempDir = null; private File logDir = null; private File confFile = null; private File readmeFile = null; // 2(tempDir?) private File dir11_1 = null; private File dir11_2 = null; private File file11_3 = null; // 3(dir11_2?) private File fileData1 = null; private File fileData2 = null; private File dirData3 = null; // private static final File cleanDir02 = new File("target/asakusa-cleaner/HDFSCleaner02"); // 1 private File fileImportData1 = null; private File fileImportData2 = null; @BeforeClass public static void setUpBeforeClass() throws Exception { UnitTestUtil.setUpBeforeClass(); UnitTestUtil.setUpEnv(); } @AfterClass public static void tearDownAfterClass() throws Exception { UnitTestUtil.tearDownAfterClass(); } @Before public void setUp() throws Exception { UnitTestUtil.startUp(); cleanDir01.mkdir(); cleanDir02.mkdir(); Properties p = ConfigurationLoader.getProperty(); p.clear(); } @After public void tearDown() throws Exception { UnitTestUtil.tearDown(); cleanDir01.delete(); cleanDir02.delete(); } /** * * <p> * execute? * * ??? * 0 * * - ??0 * - ? * * </p> * * @throws Exception */ @Test public void executeTest01() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "normal", "asakusa-cleaner", propFile }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertFalse(confFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertTrue(dir11_2.exists()); assertTrue(file11_3.exists()); // 3(dir11_2?) assertTrue(fileData1.exists()); assertTrue(fileData2.exists()); assertTrue(dirData3.exists()); // cleanDir(cleanDir01); } /** * * <p> * execute? * * ??? * 1 * * - ??0 * - ? * * </p> * * @throws Exception */ @Test public void executeTest02() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertFalse(tempDir.exists()); assertFalse(logDir.exists()); assertFalse(confFile.exists()); // 2(tempDir?) assertFalse(dir11_1.exists()); assertFalse(dir11_2.exists()); assertFalse(file11_3.exists()); // 3(dir11_2?) assertFalse(fileData1.exists()); assertFalse(fileData2.exists()); assertFalse(dirData3.exists()); // cleanDir(cleanDir01); } /** * * <p> * execute? * * ??? * 1 * * - ??3(?????????) * - * * </p> * * @throws Exception */ @Test public void executeTest03() throws Exception { // ?? createCleanDir01(); createCleanDir02(); // ? long now = new Date().getTime();// ??? long past = now - (5L * 24L * 60L * 60L * 1000L); // ? confFile.setLastModified(now); file11_3.setLastModified(past); fileData1.setLastModified(past); fileData2.setLastModified(now); fileImportData1.setLastModified(past); fileImportData2.setLastModified(past); tempDir.setLastModified(past); logDir.setLastModified(past); dir11_1.setLastModified(past); dir11_2.setLastModified(past); dirData3.setLastModified(past); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile1 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 1 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertFalse(logDir.exists()); assertTrue(confFile.exists()); // 2(tempDir?) assertFalse(dir11_1.exists()); assertTrue(dir11_2.exists()); assertFalse(file11_3.exists()); // 3(dir11_2?) assertFalse(fileData1.exists()); assertTrue(fileData2.exists()); assertFalse(dirData3.exists()); // 2 assertTrue(cleanDir02.exists()); // 1 assertFalse(fileImportData1.exists()); assertFalse(fileImportData2.exists()); // cleanDir(cleanDir01); cleanDir(cleanDir02); } /** * * <p> * execute? * * ??? * 1 * * - ??3(?????????) * - * * </p> * * @throws Exception */ @Test public void executeTest04() throws Exception { // ?? createCleanDir01(); createCleanDir02(); // ? long now = new Date().getTime();// ??? long past = now - (5L * 24L * 60L * 60L * 1000L); // ? confFile.setLastModified(past); file11_3.setLastModified(past); fileData1.setLastModified(past); fileData2.setLastModified(past); fileImportData1.setLastModified(past); fileImportData2.setLastModified(past); tempDir.setLastModified(past); logDir.setLastModified(now); dir11_1.setLastModified(now); dir11_2.setLastModified(past); dirData3.setLastModified(past); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile1 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 1 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertFalse(confFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertFalse(dir11_2.exists()); assertFalse(file11_3.exists()); // 3(dir11_2?) assertFalse(fileData1.exists()); assertFalse(fileData2.exists()); assertFalse(dirData3.exists()); // 2 assertTrue(cleanDir02.exists()); // 1 assertFalse(fileImportData1.exists()); assertFalse(fileImportData2.exists()); // cleanDir(cleanDir01); cleanDir(cleanDir02); } /** * * <p> * execute? * * ???? * * </p> * * @throws Exception */ @Test public void executeTest05() throws Exception { // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile, "1" }; HDFSCleaner cleaner = new HDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(1, result); // ?? args = new String[] {}; result = cleaner.execute(args); // ?? assertEquals(1, result); } /** * * <p> * execute? * * ??? * * </p> * * @throws Exception */ @Test public void executeTest06() throws Exception { // ?? String[] args = new String[] { "2", "test", propFile }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(1, result); } /** * * <p> * execute? * * ???? * ??????? * * </p> * * @throws Exception */ @Test public void executeTest07() throws Exception { // ?? createCleanDir02(); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile2 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(2, result); // cleanDir(cleanDir02); } /** * * <p> * execute? * * ???? * ???? * * </p> * * @throws Exception */ @Test public void executeTest08() throws Exception { // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile }; HDFSCleaner cleaner = new StubHDFSCleaner() { /* (? Javadoc) * @see com.asakusafw.bulkloader.cleaner.StubHDFSCleaner#createPath(java.lang.String) */ @Override protected Path createPath(String strCleanPath) { return new Path("hdf://localhost:1127"); } }; int result = cleaner.execute(args); // ?? assertEquals(2, result); } /** * * <p> * execute? * * ?????? * * </p> * * @throws Exception */ @Test public void executeTest09() throws Exception { // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile }; HDFSCleaner cleaner = new StubHDFSCleaner() { /* (? Javadoc) * @see com.asakusafw.bulkloader.cleaner.StubHDFSCleaner#createPath(java.lang.String) */ @Override protected Path createPath(String strCleanPath) { throw new NullPointerException(); } }; int result = cleaner.execute(args); // ?? assertEquals(1, result); } /** * * <p> * execute? * * ??? * 1 * * - ??0 * - * ??? * * </p> * * @throws Exception */ @Test public void executeTest10() throws Exception { // ?? createCleanDir01(); createCleanDir02(); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile3 }; StubHDFSCleaner cleaner = new StubHDFSCleaner(true); int result = cleaner.execute(args); // ?? assertEquals(0, result); ArrayList<String> instanceId = cleaner.getInstanceId(); assertEquals(2, instanceId.size()); assertEquals("11_1", instanceId.get(0)); assertEquals("11_2", instanceId.get(1)); // ? // 1 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertTrue(confFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertTrue(dir11_2.exists()); assertFalse(file11_3.exists()); // 3(dir11_2?) assertTrue(fileData1.exists()); assertTrue(fileData2.exists()); assertTrue(dirData3.exists()); // 2 assertTrue(cleanDir02.exists()); // 1 assertFalse(fileImportData1.exists()); assertFalse(fileImportData2.exists()); // cleanDir(cleanDir01); cleanDir(cleanDir02); } /** * * <p> * execute? * * ? * normal * * - ??0 * - .*\.txt * - ? * * </p> * * @throws Exception */ @Test public void executeTest11() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "normal", "asakusa-cleaner", propFile4 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertTrue(confFile.exists()); assertFalse(readmeFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertTrue(dir11_2.exists()); assertTrue(file11_3.exists()); // 3(dir11_2?) assertTrue(fileData1.exists()); assertTrue(fileData2.exists()); assertTrue(dirData3.exists()); // cleanDir(cleanDir01); } /** * * <p> * execute? * * ? * recursive * * - ??0 * - .*\.txt * - ? * * </p> * * @throws Exception */ @Test public void executeTest12() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "recursive", "asakusa-cleaner", propFile4 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(0, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertFalse(logDir.exists()); assertTrue(confFile.exists()); assertFalse(readmeFile.exists()); // 2(tempDir?) assertFalse(dir11_1.exists()); assertFalse(dir11_2.exists()); assertTrue(file11_3.exists()); // 3(dir11_2?) assertFalse(fileData1.exists()); assertFalse(fileData2.exists()); assertFalse(dirData3.exists()); // cleanDir(cleanDir01); } /** * * <p> * execute? * * ??? * normal * * - ??0 * - *.txt * - ? * * </p> * * @throws Exception */ @Test public void executeTest13() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "normal", "test", propFile5 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(2, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertTrue(confFile.exists()); assertTrue(readmeFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertTrue(dir11_2.exists()); assertTrue(file11_3.exists()); // 3(dir11_2?) assertTrue(fileData1.exists()); assertTrue(fileData2.exists()); assertTrue(dirData3.exists()); // cleanDir(cleanDir01); } /** * * <p> * execute? * * ?????? * normal * * - ??0 * - *.txt * - ? * * </p> * * @throws Exception */ @Test public void executeTest14() throws Exception { // ?? createCleanDir01(); // ?? String[] args = new String[] { "normal", "test", propFile6 }; HDFSCleaner cleaner = new StubHDFSCleaner(); int result = cleaner.execute(args); // ?? assertEquals(1, result); // ? // 0 assertTrue(cleanDir01.exists()); // 1 assertTrue(tempDir.exists()); assertTrue(logDir.exists()); assertTrue(confFile.exists()); assertTrue(readmeFile.exists()); // 2(tempDir?) assertTrue(dir11_1.exists()); assertTrue(dir11_2.exists()); assertTrue(file11_3.exists()); // 3(dir11_2?) assertTrue(fileData1.exists()); assertTrue(fileData2.exists()); assertTrue(dirData3.exists()); // cleanDir(cleanDir01); } /** * 1?? * @throws IOException */ private void createCleanDir01() throws IOException { cleanDir01.mkdir(); // 1 tempDir = new File(cleanDir01, "temp"); tempDir.mkdir(); logDir = new File(cleanDir01, "log"); logDir.mkdir(); confFile = new File(cleanDir01, "localfs-conf.properties"); confFile.createNewFile(); readmeFile = new File(cleanDir01, "readme.txt"); readmeFile.createNewFile(); // 2(tempDir?) dir11_1 = new File(tempDir, "11_1"); dir11_1.mkdir(); dir11_2 = new File(tempDir, "11_2"); dir11_2.mkdir(); file11_3 = new File(tempDir, "11_3.tsv"); file11_3.createNewFile(); // 3(dir11_2?) fileData1 = new File(dir11_2, "data1.txt"); fileData1.createNewFile(); fileData2 = new File(dir11_2, "data2.txt"); fileData2.createNewFile(); dirData3 = new File(dir11_2, "data3"); dirData3.mkdir(); } /** * 2?? * @throws IOException */ private void createCleanDir02() throws IOException { cleanDir02.mkdir(); // 1 fileImportData1 = new File(cleanDir02, "importdata1.tsv"); fileImportData1.createNewFile(); fileImportData2 = new File(cleanDir02, "importdata2.tsv"); fileImportData2.createNewFile(); } /** * ?? * @param cleandir * @throws IOException */ private void cleanDir(File cleandir) throws IOException { File[] listFiles = cleandir.listFiles(); for (File file : listFiles) { if (file.isFile()) { file.delete(); } if (file.isDirectory()) { FileUtils.deleteDirectory(file); } } cleandir.delete(); } } class StubHDFSCleaner extends HDFSCleaner { boolean exec = false; public StubHDFSCleaner() { super(new Configuration()); } public StubHDFSCleaner(boolean exec) { super(new Configuration()); this.exec = exec; } ArrayList<String> instanceId = new ArrayList<String>(); /** * @see com.asakusafw.cleaner.main.HDFSCleaner#createPath(java.lang.String) */ @Override protected Path createPath(String strCleanPath) { File file = new File(strCleanPath); URI uri = file.toURI(); return new Path(uri.getPath()); } /** * @see com.asakusafw.cleaner.main.HDFSCleaner#isRunningJobFlow(java.lang.String) */ @Override protected boolean isRunningJobFlow(String executionId) { instanceId.add(executionId); return exec; } /** * @return instanceId */ public ArrayList<String> getInstanceId() { Collections.sort(instanceId); return instanceId; } }