Java tutorial
/******************************************************************************* * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University * as Operator of the SLAC National Accelerator Laboratory. * Copyright (c) 2011 Brookhaven National Laboratory. * EPICS archiver appliance is distributed subject to a Software License Agreement found * in file LICENSE that is included with this distribution. *******************************************************************************/ package org.epics.archiverappliance.etl; import java.io.File; import java.sql.Timestamp; import java.util.LinkedList; import junit.framework.TestCase; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.epics.archiverappliance.Event; import org.epics.archiverappliance.EventStream; import org.epics.archiverappliance.common.BasicContext; import org.epics.archiverappliance.common.POJOEvent; import org.epics.archiverappliance.common.TimeUtils; import org.epics.archiverappliance.common.YearSecondTimestamp; import org.epics.archiverappliance.config.ArchDBRTypes; import org.epics.archiverappliance.config.ConfigServiceForTests; import org.epics.archiverappliance.config.PVTypeInfo; import org.epics.archiverappliance.config.StoragePluginURLParser; import org.epics.archiverappliance.data.ScalarValue; import org.epics.archiverappliance.engine.membuf.ArrayListEventStream; import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc; import org.epics.archiverappliance.retrieval.postprocessors.PostProcessor; import org.epics.archiverappliance.retrieval.postprocessors.PostProcessors; import org.epics.archiverappliance.retrieval.workers.CurrentThreadWorkerEventStream; import org.junit.After; import org.junit.Before; import org.junit.Test; import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin; /** * More complicated data reduction test case. * We have two PV's, one with reduced data and the other without. * We mimic data generation and ETL as it would occur in the appliance. * Finally, we compare the reduced data with the raw data + operator. * @author mshankar * */ public class DataReductionDailyETLTest extends TestCase { private static final Logger logger = Logger.getLogger(DataReductionDailyETLTest.class); String shortTermFolderName = ConfigServiceForTests.getDefaultShortTermFolder() + "/shortTerm"; String mediumTermFolderName = ConfigServiceForTests.getDefaultPBTestFolder() + "/mediumTerm"; String longTermFolderName = ConfigServiceForTests.getDefaultPBTestFolder() + "/longTerm"; private ConfigServiceForTests configService; private String rawPVName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + DataReductionDailyETLTest.class.getSimpleName(); private String reducedPVName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + DataReductionDailyETLTest.class.getSimpleName() + "reduced"; private String reduceDataUsing = "firstSample_3600"; @Before public void setUp() throws Exception { configService = new ConfigServiceForTests(new File("./bin")); if (new File(shortTermFolderName).exists()) { FileUtils.deleteDirectory(new File(shortTermFolderName)); } if (new File(mediumTermFolderName).exists()) { FileUtils.deleteDirectory(new File(mediumTermFolderName)); } if (new File(longTermFolderName).exists()) { FileUtils.deleteDirectory(new File(longTermFolderName)); } } @After public void tearDown() throws Exception { if (new File(shortTermFolderName).exists()) { FileUtils.deleteDirectory(new File(shortTermFolderName)); } if (new File(mediumTermFolderName).exists()) { FileUtils.deleteDirectory(new File(mediumTermFolderName)); } if (new File(longTermFolderName).exists()) { FileUtils.deleteDirectory(new File(longTermFolderName)); } } /** * 1) Set up the raw and reduced PV's * 2) Generate data in STS * 3) Run ETL * 4) Compare */ @Test public void testReducedETL() throws Exception { // Set up the raw and reduced PV's PlainPBStoragePlugin etlSTS = (PlainPBStoragePlugin) StoragePluginURLParser .parseStoragePlugin("pb://localhost?name=STS&rootFolder=" + shortTermFolderName + "/&partitionGranularity=PARTITION_HOUR", configService); ; PlainPBStoragePlugin etlMTS = (PlainPBStoragePlugin) StoragePluginURLParser .parseStoragePlugin("pb://localhost?name=MTS&rootFolder=" + mediumTermFolderName + "/&partitionGranularity=PARTITION_DAY", configService); PlainPBStoragePlugin etlLTS = (PlainPBStoragePlugin) StoragePluginURLParser .parseStoragePlugin( "pb://localhost?name=LTS&rootFolder=" + longTermFolderName + "/&partitionGranularity=PARTITION_YEAR&reducedata=" + reduceDataUsing, configService); { PVTypeInfo typeInfo = new PVTypeInfo(rawPVName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1); String[] dataStores = new String[] { etlSTS.getURLRepresentation(), etlMTS.getURLRepresentation(), etlLTS.getURLRepresentation() }; typeInfo.setDataStores(dataStores); configService.updateTypeInfoForPV(rawPVName, typeInfo); configService.registerPVToAppliance(rawPVName, configService.getMyApplianceInfo()); } { PVTypeInfo typeInfo = new PVTypeInfo(reducedPVName, ArchDBRTypes.DBR_SCALAR_DOUBLE, true, 1); String[] dataStores = new String[] { etlSTS.getURLRepresentation(), etlMTS.getURLRepresentation(), etlLTS.getURLRepresentation() }; typeInfo.setDataStores(dataStores); configService.updateTypeInfoForPV(reducedPVName, typeInfo); configService.registerPVToAppliance(reducedPVName, configService.getMyApplianceInfo()); } // Control ETL manually configService.getETLLookup().manualControlForUnitTests(); short currentYear = TimeUtils.getCurrentYear(); for (int day = 0; day < 365; day++) { // Generate data into the STS on a daily basis ArrayListEventStream genDataRaw = new ArrayListEventStream(86400, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, rawPVName, currentYear)); ArrayListEventStream genDataReduced = new ArrayListEventStream(86400, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, reducedPVName, currentYear)); for (int second = 0; second < 86400; second++) { YearSecondTimestamp ysts = new YearSecondTimestamp(currentYear, day * 86400 + second, 0); Timestamp ts = TimeUtils.convertFromYearSecondTimestamp(ysts); genDataRaw.add(new POJOEvent(ArchDBRTypes.DBR_SCALAR_DOUBLE, ts, new ScalarValue<Double>(second * 1.0), 0, 0)); genDataReduced.add(new POJOEvent(ArchDBRTypes.DBR_SCALAR_DOUBLE, ts, new ScalarValue<Double>(second * 1.0), 0, 0)); } try (BasicContext context = new BasicContext()) { etlSTS.appendData(context, rawPVName, genDataRaw); etlSTS.appendData(context, reducedPVName, genDataReduced); } logger.debug("Done generating data into the STS for day " + day); // Run ETL at the end of the day Timestamp timeETLruns = TimeUtils .convertFromYearSecondTimestamp(new YearSecondTimestamp(currentYear, day * 86400 + 86399, 0)); ETLExecutor.runETLs(configService, timeETLruns); logger.debug("Done performing ETL as though today is " + TimeUtils.convertToHumanReadableString(timeETLruns)); // Compare data for raw+postprocessor and reduced PV's. PostProcessor postProcessor = PostProcessors.findPostProcessor(reduceDataUsing); postProcessor.initialize(reduceDataUsing, rawPVName); int rawWithPPCount = 0; int reducedCount = 0; try (BasicContext context = new BasicContext()) { Timestamp startTime = TimeUtils.minusDays(TimeUtils.now(), 10 * 366); Timestamp endTime = TimeUtils.plusDays(TimeUtils.now(), 10 * 366); LinkedList<Timestamp> rawTimestamps = new LinkedList<Timestamp>(); LinkedList<Timestamp> reducedTimestamps = new LinkedList<Timestamp>(); try (EventStream rawWithPP = new CurrentThreadWorkerEventStream(rawPVName, etlLTS.getDataForPV(context, rawPVName, startTime, endTime, postProcessor))) { for (Event e : rawWithPP) { rawTimestamps.add(e.getEventTimeStamp()); rawWithPPCount++; } } try (EventStream reduced = new CurrentThreadWorkerEventStream(reducedPVName, etlLTS.getDataForPV(context, reducedPVName, startTime, endTime))) { for (Event e : reduced) { reducedTimestamps.add(e.getEventTimeStamp()); reducedCount++; } } logger.debug("For day " + day + " we have " + rawWithPPCount + " rawWithPP events and " + reducedCount + " reduced events"); if (rawTimestamps.size() != reducedTimestamps.size()) { while (!rawTimestamps.isEmpty() || !reducedTimestamps.isEmpty()) { if (!rawTimestamps.isEmpty()) logger.info("Raw/PP " + TimeUtils.convertToHumanReadableString(rawTimestamps.pop())); if (!reducedTimestamps.isEmpty()) logger.info( "Reduced" + TimeUtils.convertToHumanReadableString(reducedTimestamps.pop())); } } assertTrue("For day " + day + " we have " + rawWithPPCount + " rawWithPP events and " + reducedCount + " reduced events", rawWithPPCount == reducedCount); } if (day > 2) { assertTrue("For day " + day + ", seems like no events were moved by ETL into LTS for " + rawPVName + " Count = " + rawWithPPCount, (rawWithPPCount != 0)); assertTrue("For day " + day + ", seems like no events were moved by ETL into LTS for " + reducedPVName + " Count = " + reducedCount, (reducedCount != 0)); } } } }