Java tutorial
/******************************************************************************* * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University * as Operator of the SLAC National Accelerator Laboratory. * Copyright (c) 2011 Brookhaven National Laboratory. * EPICS archiver appliance is distributed subject to a Software License Agreement found * in file LICENSE that is included with this distribution. *******************************************************************************/ package org.epics.archiverappliance.etl; import java.io.File; import java.nio.file.Path; import java.sql.Timestamp; import java.util.LinkedList; import java.util.List; import junit.framework.TestCase; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.epics.archiverappliance.Event; import org.epics.archiverappliance.EventStream; import org.epics.archiverappliance.common.BasicContext; import org.epics.archiverappliance.common.TimeUtils; import org.epics.archiverappliance.config.ArchDBRTypes; import org.epics.archiverappliance.config.ConfigServiceForTests; import org.epics.archiverappliance.config.PVTypeInfo; import org.epics.archiverappliance.config.StoragePluginURLParser; import org.epics.archiverappliance.retrieval.postprocessors.PostProcessor; import org.epics.archiverappliance.retrieval.postprocessors.PostProcessors; import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream; import org.epics.archiverappliance.utils.nio.ArchPaths; import org.epics.archiverappliance.utils.simulation.SimulationEventStream; import org.epics.archiverappliance.utils.simulation.SineGenerator; import org.junit.After; import org.junit.Before; import org.junit.Test; import edu.stanford.slac.archiverappliance.PlainPB.PlainPBPathNameUtility; import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin; import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin.CompressionMode; import edu.stanford.slac.archiverappliance.PlainPB.utils.ValidatePBFile; /** * Test data reduction as part of ETL. * @author mshankar * */ public class DataReductionTest extends TestCase { private static final Logger logger = Logger.getLogger(DataReductionTest.class); String shortTermFolderName = ConfigServiceForTests.getDefaultShortTermFolder() + "/shortTerm"; String mediumTermFolderName = ConfigServiceForTests.getDefaultPBTestFolder() + "/mediumTerm"; private ConfigServiceForTests configService; @Before public void setUp() throws Exception { configService = new ConfigServiceForTests(new File("./bin")); if (new File(shortTermFolderName).exists()) { FileUtils.deleteDirectory(new File(shortTermFolderName)); } if (new File(mediumTermFolderName).exists()) { FileUtils.deleteDirectory(new File(mediumTermFolderName)); } } @After public void tearDown() throws Exception { if (new File(shortTermFolderName).exists()) { FileUtils.deleteDirectory(new File(shortTermFolderName)); } if (new File(mediumTermFolderName).exists()) { FileUtils.deleteDirectory(new File(mediumTermFolderName)); } } /** * Generates some data in STS; then calls the ETL to move it to MTS. * Check that we only move reduced data into the MTS. */ @Test public void testMove() throws Exception { String reduceDataUsing = "firstSample_3600"; PlainPBStoragePlugin etlSrc = (PlainPBStoragePlugin) StoragePluginURLParser .parseStoragePlugin("pb://localhost?name=STS&rootFolder=" + shortTermFolderName + "/&partitionGranularity=PARTITION_DAY", configService); ; PlainPBStoragePlugin etlDest = (PlainPBStoragePlugin) StoragePluginURLParser .parseStoragePlugin( "pb://localhost?name=MTS&rootFolder=" + mediumTermFolderName + "/&partitionGranularity=PARTITION_YEAR&reducedata=" + reduceDataUsing, configService); logger.info("Testing data reduction for " + etlSrc.getPartitionGranularity() + " to " + etlDest.getPartitionGranularity()); String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + "ETL_testMove" + etlSrc.getPartitionGranularity(); SimulationEventStream simstream = new SimulationEventStream(ArchDBRTypes.DBR_SCALAR_DOUBLE, new SineGenerator(0)); try (BasicContext context = new BasicContext()) { etlSrc.appendData(context, pvName, simstream); } logger.info("Done creating src data for PV " + pvName); long beforeCount = 0; try (BasicContext context = new BasicContext(); EventStream before = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, TimeUtils.minusDays(TimeUtils.now(), 366), TimeUtils.plusDays(TimeUtils.now(), 366)))) { for (@SuppressWarnings("unused") Event e : before) { beforeCount++; } } PostProcessor postProcessor = PostProcessors.findPostProcessor(reduceDataUsing); postProcessor.initialize(reduceDataUsing, pvName); long beforeReducedCount = 0; List<Event> reducedEvents = new LinkedList<Event>(); try (BasicContext context = new BasicContext(); EventStream before = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, TimeUtils.minusDays(TimeUtils.now(), 366), TimeUtils.plusDays(TimeUtils.now(), 366), postProcessor))) { for (Event e : before) { reducedEvents.add(e.makeClone()); beforeReducedCount++; } } PVTypeInfo typeInfo = new PVTypeInfo(pvName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1); String[] dataStores = new String[] { etlSrc.getURLRepresentation(), etlDest.getURLRepresentation() }; typeInfo.setDataStores(dataStores); configService.updateTypeInfoForPV(pvName, typeInfo); configService.registerPVToAppliance(pvName, configService.getMyApplianceInfo()); configService.getETLLookup().manualControlForUnitTests(); Timestamp timeETLruns = TimeUtils.plusDays(TimeUtils.now(), 365 * 10); ETLExecutor.runETLs(configService, timeETLruns); logger.info( "Done performing ETL as though today is " + TimeUtils.convertToHumanReadableString(timeETLruns)); Timestamp startOfRequest = TimeUtils.minusDays(TimeUtils.now(), 366); Timestamp endOfRequest = TimeUtils.plusDays(TimeUtils.now(), 366); // Check that all the files in the destination store are valid files. Path[] allPaths = PlainPBPathNameUtility.getAllPathsForPV(new ArchPaths(), etlDest.getRootFolder(), pvName, ".pb", etlDest.getPartitionGranularity(), CompressionMode.NONE, configService.getPVNameToKeyConverter()); assertTrue("PlainPBFileNameUtility returns null for getAllFilesForPV for " + pvName, allPaths != null); assertTrue("PlainPBFileNameUtility returns empty array for getAllFilesForPV for " + pvName + " when looking in " + etlDest.getRootFolder(), allPaths.length > 0); for (Path destPath : allPaths) { assertTrue("File validation failed for " + destPath.toAbsolutePath().toString(), ValidatePBFile.validatePBFile(destPath, false)); } logger.info("Asking for data between" + TimeUtils.convertToHumanReadableString(startOfRequest) + " and " + TimeUtils.convertToHumanReadableString(endOfRequest)); long afterCount = 0; try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) { assertNotNull(afterDest); for (@SuppressWarnings("unused") Event e : afterDest) { afterCount++; } } logger.info("Of the " + beforeCount + " events, " + afterCount + " events were moved into the dest store."); assertTrue("Seems like no events were moved by ETL " + afterCount, (afterCount != 0)); long afterSourceCount = 0; try (BasicContext context = new BasicContext(); EventStream afterSrc = new CurrentThreadWorkerEventStream(pvName, etlSrc.getDataForPV(context, pvName, startOfRequest, endOfRequest))) { for (@SuppressWarnings("unused") Event e : afterSrc) { afterSourceCount++; } } assertTrue("Seems like we still have " + afterSourceCount + " events in the source ", (afterSourceCount == 0)); // Now compare the events itself try (BasicContext context = new BasicContext(); EventStream afterDest = new CurrentThreadWorkerEventStream(pvName, etlDest.getDataForPV(context, pvName, startOfRequest, endOfRequest))) { int index = 0; for (Event afterEvent : afterDest) { Event beforeEvent = reducedEvents.get(index); assertTrue( "Before timestamp " + TimeUtils.convertToHumanReadableString(beforeEvent.getEventTimeStamp()) + " After timestamp " + TimeUtils.convertToHumanReadableString(afterEvent.getEventTimeStamp()), beforeEvent.getEventTimeStamp().equals(afterEvent.getEventTimeStamp())); assertTrue( "Before value " + beforeEvent.getSampleValue().getValue() + " After value " + afterEvent.getSampleValue().getValue(), beforeEvent.getSampleValue().getValue().equals(afterEvent.getSampleValue().getValue())); index++; } } assertTrue("Of the total " + beforeCount + " event, we should have moved " + beforeReducedCount + ". Instead we seem to have moved " + afterCount, beforeReducedCount == afterCount); } }