Java tutorial
/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.addthis.hydra.task.map; import java.io.File; import java.io.IOException; import java.util.List; import com.addthis.basis.util.LessFiles; import org.apache.commons.io.FileUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; public class DataPurgeServiceImplTest { private static final Logger logger = LoggerFactory.getLogger(DataPurgeServiceImplTest.class); private File tmpPrefix; DataPurgeServiceImpl dataPurgeService; public String getPrefix() { return tmpPrefix + "/purgetest"; //return "/tmp/purgetest"; } @Before public void before() throws IOException { tmpPrefix = LessFiles.createTempDir(); // = /tmp dataPurgeService = new DataPurgeServiceImpl(); } @After public void after() { FileUtils.deleteQuietly(tmpPrefix); } @Test public void testArgumentValidation() throws Exception { DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setMaxAgeInDays(1); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setDirectoryPrefix(new String[] { "base" }); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setDatePathFormat("yy/MM/dd"); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, null)); dataPurgeConfig.setMaxAgeInDays(-1); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setMaxAgeInDays(1); dataPurgeConfig.setDatePathFormat(null); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setDatePathFormat("yy/MM/dd"); dataPurgeConfig.setFileBasedPurge(true); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setDateStartIndex(2); assertFalse(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); dataPurgeConfig.setDateStringLength(4); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, new DateTime())); } @Test public void testExpandPrefix() throws Exception { String prefix = getPrefix() + "/dptest/"; createNovDecData(prefix); String test1 = prefix + "*"; String test2 = prefix + "*/*"; String test3 = prefix + "*/*/*"; String test4 = prefix + "11/01/*"; String test5 = prefix + "*/01"; String test6 = prefix + "*/01/*"; String test7 = prefix + "*1/*/01"; List<File> result1 = dataPurgeService.expandPrefix(test1); List<File> result2 = dataPurgeService.expandPrefix(test2); List<File> result3 = dataPurgeService.expandPrefix(test3); List<File> result4 = dataPurgeService.expandPrefix(test4); List<File> result5 = dataPurgeService.expandPrefix(test5); List<File> result6 = dataPurgeService.expandPrefix(test6); List<File> result7 = dataPurgeService.expandPrefix(test7); assertEquals(result1.size(), 2); assertEquals(result2.size(), 4); assertEquals(result3.size(), 12); assertEquals(result4.size(), 3); assertEquals(result5.size(), 2); assertEquals(result6.size(), 6); assertEquals(result7.size(), 2); } @Test public void testGetDirectoryList() throws IOException { String prefix = getPrefix() + "/dptest"; createData(prefix); List<File> directoryList = dataPurgeService.getSubdirectoryList(new File(prefix), null); assertNotNull(directoryList); assertEquals(9, directoryList.size()); } @Test public void testPurgeData() throws IOException { String prefix = getPrefix() + "/dptest"; createData(prefix); DateTimeFormatter formatter = DateTimeFormat.forPattern("yy/MM/dd"); DateTime currentTime = formatter.parseDateTime("11/01/04"); DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setMaxAgeInDays(1); dataPurgeConfig.setDirectoryPrefix(new String[] { prefix }); dataPurgeConfig.setDatePathFormat("yy/MM/dd"); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, currentTime)); List<File> directoryList = dataPurgeService.getSubdirectoryList(new File(prefix), null); assertNotNull(directoryList); assertEquals(7, directoryList.size()); } @Test public void testWildcardPurgeData() throws IOException { String tmp = getPrefix(); String prefix = tmp + "/split/search-*-hashed/*"; createData(tmp + "/split/search-us-hashed/0"); createData(tmp + "/split/search-us-hashed/1"); createData(tmp + "/split/search-gb-hashed/0"); createData(tmp + "/split/search-gb-hashed/1"); DateTimeFormatter formatter = DateTimeFormat.forPattern("yy/MM/dd"); DateTime currentTime = formatter.parseDateTime("11/01/04"); DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setMaxAgeInDays(1); dataPurgeConfig.setDirectoryPrefix(new String[] { prefix }); dataPurgeConfig.setDatePathFormat("yy/MM/dd"); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, currentTime)); List<File> directoryList = dataPurgeService.generateDirectoryList(prefix); assertNotNull(directoryList); assertEquals(28, directoryList.size()); } @Test public void testPurgeDataHourly() throws IOException { String prefix = getPrefix() + "/ht"; createHourlyData(prefix); DateTimeFormatter formatter = DateTimeFormat.forPattern("yyMMdd/HH"); DateTime currentTime = formatter.parseDateTime("120326/13"); DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setCleanEmptyParents(true); dataPurgeConfig.setMaxAgeInHours(22); dataPurgeConfig.setDirectoryPrefix(new String[] { prefix }); dataPurgeConfig.setDatePathFormat("yyMMdd/HH"); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, currentTime)); List<File> directoryList = dataPurgeService.getSubdirectoryList(new File(prefix), null); logger.trace("remaining after hourly purge {}", directoryList); assertNotNull(directoryList); assertEquals(6, directoryList.size()); // includes sub-directories } @Test public void testPurgeData_Files() throws IOException { String prefix = getPrefix() + "/dptest"; createFileBasedData(prefix); DateTimeFormatter formatter = DateTimeFormat.forPattern("yyMMdd"); DateTime currentTime = formatter.parseDateTime("110104"); DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setMaxAgeInDays(1); dataPurgeConfig.setDirectoryPrefix(new String[] { prefix }); dataPurgeConfig.setDatePathFormat("yyMMdd"); dataPurgeConfig.setFileBasedPurge(true); dataPurgeConfig.setDateStartIndex(9); dataPurgeConfig.setDateStringLength(6); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, currentTime)); List<File> directoryList = dataPurgeService.getSubdirectoryList(new File(prefix), null); assertNotNull(directoryList); assertEquals(1, directoryList.size()); File[] fileList = directoryList.iterator().next().listFiles(); assertNotNull(fileList); assertEquals(fileList.length, 4); } @Test public void testPurgeData_Files_TwoPrefixDirs() throws IOException { String prefix = getPrefix() + "/dptest"; String prefix2 = getPrefix() + "/dptest2"; createFileBasedData(prefix); createFileBasedData(prefix2); DateTimeFormatter formatter = DateTimeFormat.forPattern("yyMMdd"); DateTime currentTime = formatter.parseDateTime("110104"); DataPurgeConfig dataPurgeConfig = new DataPurgeConfig(); dataPurgeConfig.setMaxAgeInDays(1); dataPurgeConfig.setDirectoryPrefix(new String[] { prefix, prefix2 }); dataPurgeConfig.setDatePathFormat("yyMMdd"); dataPurgeConfig.setFileBasedPurge(true); dataPurgeConfig.setDateStartIndex(9); dataPurgeConfig.setDateStringLength(6); assertTrue(dataPurgeService.purgeData(dataPurgeConfig, currentTime)); List<File> directoryList = dataPurgeService.getSubdirectoryList(new File(prefix), null); assertNotNull(directoryList); assertEquals(1, directoryList.size()); File[] fileList = directoryList.iterator().next().listFiles(); assertNotNull(fileList); assertEquals(fileList.length, 4); directoryList = dataPurgeService.getSubdirectoryList(new File(prefix2), null); assertNotNull(directoryList); assertEquals(1, directoryList.size()); fileList = directoryList.iterator().next().listFiles(); assertNotNull(fileList); assertEquals(fileList.length, 4); } @Test public void testShouldDelete() throws IOException { String prefix = getPrefix() + "/dptest"; createData(prefix); DateTimeFormatter formatter = DateTimeFormat.forPattern("yy/MM/dd"); DateTime oldest = formatter.parseDateTime("11/01/03"); assertTrue(dataPurgeService.shouldDelete(formatter, oldest, "11/01/01")); assertTrue(dataPurgeService.shouldDelete(formatter, oldest, "11/01/02")); assertFalse(dataPurgeService.shouldDelete(formatter, oldest, "11/01/03")); assertFalse(dataPurgeService.shouldDelete(formatter, oldest, "11/01/04")); assertTrue(dataPurgeService.shouldDelete(formatter, oldest, "10/01/04")); assertFalse(dataPurgeService.shouldDelete(formatter, oldest, "11/04/04")); assertFalse(dataPurgeService.shouldDelete(formatter, oldest, "12/04/04")); } private void createData(String prefix) throws IOException { new File(prefix + "/11/01/01").mkdirs(); new File(prefix + "/11/01/01/testFile.txt").createNewFile(); new File(prefix + "/11/01/02").mkdirs(); new File(prefix + "/11/01/02/testFile.txt").createNewFile(); new File(prefix + "/11/01/03").mkdirs(); new File(prefix + "/11/01/03/testFile.txt").createNewFile(); new File(prefix + "/11/01/04").mkdirs(); new File(prefix + "/11/01/04/testFile.txt").createNewFile(); new File(prefix + "/11/01/05").mkdirs(); new File(prefix + "/11/01/05/testFile.txt").createNewFile(); new File(prefix + "/11/01/06").mkdirs(); new File(prefix + "/11/01/06/testFile.txt").createNewFile(); } private void createNovDecData(String prefix) throws IOException { new File(prefix + "/11/01/01").mkdirs(); new File(prefix + "/11/01/01/testFile.txt").createNewFile(); new File(prefix + "/11/01/02").mkdirs(); new File(prefix + "/11/01/02/testFile.txt").createNewFile(); new File(prefix + "/11/01/03").mkdirs(); new File(prefix + "/11/01/03/testFile.txt").createNewFile(); new File(prefix + "/11/02/01").mkdirs(); new File(prefix + "/11/02/01/testFile.txt").createNewFile(); new File(prefix + "/11/02/02").mkdirs(); new File(prefix + "/11/02/02/testFile.txt").createNewFile(); new File(prefix + "/11/02/03").mkdirs(); new File(prefix + "/11/02/03/testFile.txt").createNewFile(); new File(prefix + "/12/01/01").mkdirs(); new File(prefix + "/12/01/01/testFile.txt").createNewFile(); new File(prefix + "/12/01/02").mkdirs(); new File(prefix + "/12/01/02/testFile.txt").createNewFile(); new File(prefix + "/12/01/03").mkdirs(); new File(prefix + "/12/01/03/testFile.txt").createNewFile(); new File(prefix + "/12/02/01").mkdirs(); new File(prefix + "/12/02/01/testFile.txt").createNewFile(); new File(prefix + "/12/02/02").mkdirs(); new File(prefix + "/12/02/02/testFile.txt").createNewFile(); new File(prefix + "/12/02/03").mkdirs(); new File(prefix + "/12/02/03/testFile.txt").createNewFile(); } private void createHourlyData(String prefix) throws IOException { new File(prefix + "/120324/12").mkdirs(); new File(prefix + "/120324/12/testFile.txt").createNewFile(); new File(prefix + "/120325/12").mkdirs(); new File(prefix + "/120325/12/testFile.txt").createNewFile(); new File(prefix + "/120325/13").mkdirs(); new File(prefix + "/120325/13/testFile.txt").createNewFile(); new File(prefix + "/120325/14").mkdirs(); new File(prefix + "/120325/14/testFile.txt").createNewFile(); new File(prefix + "/120325/15").mkdirs(); new File(prefix + "/120325/15/testFile.txt").createNewFile(); new File(prefix + "/120325/16").mkdirs(); new File(prefix + "/120325/16/testFile.txt").createNewFile(); new File(prefix + "/120326/08").mkdirs(); new File(prefix + "/120326/08/testFile.txt").createNewFile(); } private void createFileBasedData(String prefix) throws IOException { new File(prefix).mkdirs(); new File(prefix + "/testFile-110101.txt").createNewFile(); new File(prefix + "/testFile-110102.txt").createNewFile(); new File(prefix + "/testFile-110103.txt").createNewFile(); new File(prefix + "/testFile-110104.txt").createNewFile(); new File(prefix + "/testFile-110105.txt").createNewFile(); new File(prefix + "/testFile-110106.txt").createNewFile(); } }