Java tutorial
/******************************************************************************* * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University * as Operator of the SLAC National Accelerator Laboratory. * Copyright (c) 2011 Brookhaven National Laboratory. * EPICS archiver appliance is distributed subject to a Software License Agreement found * in file LICENSE that is included with this distribution. *******************************************************************************/ package org.epics.archiverappliance.retrieval.client; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; import java.sql.Timestamp; import java.util.LinkedList; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.epics.archiverappliance.Event; import org.epics.archiverappliance.EventStream; import org.epics.archiverappliance.TomcatSetup; import org.epics.archiverappliance.common.BasicContext; import org.epics.archiverappliance.common.TimeUtils; import org.epics.archiverappliance.common.YearSecondTimestamp; import org.epics.archiverappliance.config.ArchDBRTypes; import org.epics.archiverappliance.config.ConfigServiceForTests; import org.epics.archiverappliance.data.ScalarValue; import org.epics.archiverappliance.engine.membuf.ArrayListEventStream; import org.epics.archiverappliance.retrieval.RemotableEventStreamDesc; import org.epics.archiverappliance.utils.simulation.SimulationEvent; import org.junit.After; import org.junit.Before; import org.junit.Test; import edu.stanford.slac.archiverappliance.PB.data.PBCommonSetup; import edu.stanford.slac.archiverappliance.PlainPB.PlainPBStoragePlugin; /** * Test retrieval across year spans when some of the data is missing. * We generate data for these time periods * <ol> * <li>Sep 2011 - Oct 2011</li> * <li>Jun 2012 - Jul 2012</li> * </ol> * * We then make requests for various time periods and check the first sample and number of samples. * * @author mshankar * */ public class MissingDataYearSpanRetrievalTest { private static Logger logger = Logger.getLogger(MissingDataYearSpanRetrievalTest.class.getName()); String testSpecificFolder = "MissingDataYearSpanRetrieval"; String pvName = ConfigServiceForTests.ARCH_UNIT_TEST_PVNAME_PREFIX + ":" + testSpecificFolder + ":mdata_yspan"; File dataFolder = new File(ConfigServiceForTests.getDefaultPBTestFolder() + File.separator + "ArchUnitTest" + File.separator + testSpecificFolder); PBCommonSetup pbSetup = new PBCommonSetup(); PlainPBStoragePlugin pbplugin = new PlainPBStoragePlugin(); TomcatSetup tomcatSetup = new TomcatSetup(); private LinkedList<Timestamp> generatedTimeStamps = new LinkedList<Timestamp>(); @Before public void setUp() throws Exception { pbSetup.setUpRootFolder(pbplugin); logger.info("Data folder is " + dataFolder.getAbsolutePath()); FileUtils.deleteDirectory(dataFolder); generateData(); tomcatSetup.setUpWebApps(this.getClass().getSimpleName()); } private void generateData() throws IOException { { // Generate some data for Sep 2011 - Oct 2011, one per day Timestamp sep2011 = TimeUtils.convertFromISO8601String("2011-09-01T00:00:00.000Z"); int sep201101secsIntoYear = TimeUtils.getSecondsIntoYear(TimeUtils.convertToEpochSeconds(sep2011)); short year = 2011; ArrayListEventStream strm = new ArrayListEventStream(0, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, year)); for (int day = 0; day < 30; day++) { strm.add(new SimulationEvent(sep201101secsIntoYear + day * 86400, year, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>(0.0))); generatedTimeStamps.add(TimeUtils.convertFromYearSecondTimestamp( new YearSecondTimestamp(year, sep201101secsIntoYear + day * 86400, 0))); } try (BasicContext context = new BasicContext()) { pbplugin.appendData(context, pvName, strm); } } { // Generate some data for Jun 2012 - Jul 2012, one per day Timestamp jun2012 = TimeUtils.convertFromISO8601String("2012-06-01T00:00:00.000Z"); int jun201201secsIntoYear = TimeUtils.getSecondsIntoYear(TimeUtils.convertToEpochSeconds(jun2012)); short year = 2012; ArrayListEventStream strm = new ArrayListEventStream(0, new RemotableEventStreamDesc(ArchDBRTypes.DBR_SCALAR_DOUBLE, pvName, year)); for (int day = 0; day < 30; day++) { strm.add(new SimulationEvent(jun201201secsIntoYear + day * 86400, year, ArchDBRTypes.DBR_SCALAR_DOUBLE, new ScalarValue<Double>(0.0))); generatedTimeStamps.add(TimeUtils.convertFromYearSecondTimestamp( new YearSecondTimestamp(year, jun201201secsIntoYear + day * 86400, 0))); } try (BasicContext context = new BasicContext()) { pbplugin.appendData(context, pvName, strm); } } } @After public void tearDown() throws Exception { tomcatSetup.tearDown(); FileUtils.deleteDirectory(dataFolder); } /** * <pre> * .....Sep,2011.....Oct,2011..............Jan,1,2012..........Jun,2012......Jul,2012...............Dec,2012...... * [] - should return no data * ...[.....] should return data whose first value should be Sep 1, 2011 * ............[.....] should return data whose first value is start time - 1 * .................[...........] should return data whose first value is start time - 1 * ...................................[.] should return one sample for the last day of Sept, 2011 * ...................................[...................] should return one sample for the last day of Sept, 2011 * ................................................[..] should return one sample for the last day of Sept, 2011 * ................................................[...............] should return may samples with the first sample as the last day of Sept, 2011 * ..................................................................[......] should return may samples all from 2012 * ..........................................................................................[..] should return one sample for the last day of Jun, 2012 * ...........................................................................................................................[..] should return one sample for the last day of Jun, 2012 * <pre> * @throws IOException */ @Test public void testMissingDataYearSpan() throws Exception { testRetrieval("2011-06-01T00:00:00.000Z", "2011-07-01T00:00:00.000Z", 0, null, -1, "Before all data"); testRetrieval("2011-08-10T00:00:00.000Z", "2011-09-15T10:00:00.000Z", 15, "2011-09-01T00:00:00.000Z", 0, "Aug/10/2011 - Sep/15/2011"); testRetrieval("2011-09-10T00:00:00.000Z", "2011-09-15T10:00:00.000Z", 6, "2011-09-09T00:00:00.000Z", 8, "Sep/10/2011 - Sep/15/2011"); testRetrieval("2011-09-10T00:00:00.000Z", "2011-10-15T10:00:00.000Z", 22, "2011-09-09T00:00:00.000Z", 8, "Sep/10/2011 - Oct/15/2011"); testRetrieval("2011-10-10T00:00:00.000Z", "2011-10-15T10:00:00.000Z", 1, "2011-09-30T00:00:00.000Z", 29, "Oct/10/2011 - Oct/15/2011"); testRetrieval("2011-10-10T00:00:00.000Z", "2012-01-15T10:00:00.000Z", 1, "2011-09-30T00:00:00.000Z", 29, "Oct/10/2011 - Jan/15/2012"); testRetrieval("2012-01-10T00:00:00.000Z", "2012-01-15T10:00:00.000Z", 1, "2011-09-30T00:00:00.000Z", 29, "Jan/10/2012 - Jan/15/2012"); testRetrieval("2012-01-10T00:00:00.000Z", "2012-06-15T10:00:00.000Z", 16, "2011-09-30T00:00:00.000Z", 29, "Jan/10/2012 - Jun/15/2012"); testRetrieval("2012-06-10T00:00:00.000Z", "2012-06-15T10:00:00.000Z", 6, "2012-06-09T00:00:00.000Z", 38, "Jun/10/2012 - Jun/15/2012"); testRetrieval("2012-09-10T00:00:00.000Z", "2012-09-15T10:00:00.000Z", 1, "2012-06-30T00:00:00.000Z", 59, "Sep/10/2012 - Sep/15/2012"); testRetrieval("2013-01-10T00:00:00.000Z", "2013-01-15T10:00:00.000Z", 1, "2012-06-30T00:00:00.000Z", 59, "Jan/10/2013 - Jan/15/2013"); // logger.info("Try now..."); // try { Thread.sleep(300*1000); } catch(Throwable t) {} // logRetrieval("2011-10-10T00:00:00.000Z", "2011-10-15T10:00:00.000Z", new File("/tmp/Oct10.dat")); } /** * @param startStr - Start time of request * @param endStr - End time of request * @param expectedMinEventCount - How many events we expect at a minimum * @param firstTimeStampExpectedStr - The time stamp of the first event * @param firstTSIndex - If present, the index into generatedTimeStamps for the first event. Set to -1 if you want to skip this check. * @param msg - msg to add to log msgs and the like * @throws IOException */ private void testRetrieval(String startStr, String endStr, int expectedMinEventCount, String firstTimeStampExpectedStr, int firstTSIndex, String msg) throws IOException { Timestamp start = TimeUtils.convertFromISO8601String(startStr); Timestamp end = TimeUtils.convertFromISO8601String(endStr); Timestamp firstTimeStampExpected = null; if (firstTimeStampExpectedStr != null) { firstTimeStampExpected = TimeUtils.convertFromISO8601String(firstTimeStampExpectedStr); } if (firstTSIndex != -1) { assertTrue("Incorrect specification - Str is " + firstTimeStampExpectedStr + " and from array " + TimeUtils.convertToISO8601String(generatedTimeStamps.get(firstTSIndex)) + " for " + msg, firstTimeStampExpected.equals(generatedTimeStamps.get(firstTSIndex))); } RawDataRetrievalAsEventStream rawDataRetrieval = new RawDataRetrievalAsEventStream( "http://localhost:" + ConfigServiceForTests.RETRIEVAL_TEST_PORT + "/retrieval/data/getData.raw"); Timestamp obtainedFirstSample = null; int eventCount = 0; try (EventStream stream = rawDataRetrieval.getDataForPVS(new String[] { pvName }, start, end, null)) { if (stream != null) { for (Event e : stream) { if (obtainedFirstSample == null) { obtainedFirstSample = e.getEventTimeStamp(); } assertTrue( "Expecting sample with timestamp " + TimeUtils.convertToISO8601String( generatedTimeStamps.get(firstTSIndex + eventCount)) + " got " + TimeUtils.convertToISO8601String(e.getEventTimeStamp()) + " for " + msg, e.getEventTimeStamp().equals(generatedTimeStamps.get(firstTSIndex + eventCount))); eventCount++; } } else { logger.info("Stream is null for " + msg); } } assertTrue("Expecting at least " + expectedMinEventCount + " got " + eventCount + " for " + msg, eventCount >= expectedMinEventCount); if (firstTimeStampExpected != null) { if (obtainedFirstSample == null) { fail("Expecting at least one value for " + msg); } else { assertTrue( "Expecting first sample to be " + TimeUtils.convertToISO8601String(firstTimeStampExpected) + " got " + TimeUtils.convertToISO8601String(obtainedFirstSample) + " for " + msg, firstTimeStampExpected.equals(obtainedFirstSample)); } } else { if (obtainedFirstSample != null) { fail("Expecting no values for " + msg + " Got value from " + TimeUtils.convertToISO8601String(obtainedFirstSample)); } } } // private void logRetrieval(String startStr, String endStr, File dataFile) throws Exception { // StringWriter buf = new StringWriter(); // buf.append("http://localhost:" + ConfigServiceForTests.RETRIEVAL_TEST_PORT+ "/retrieval/data/getData.raw") // .append("?pv=").append(pvName) // .append("&from=").append(startStr) // .append("&to=").append(endStr); // String getURL = buf.toString(); // logger.info("URL to fetch data is " + getURL); // URL url = new URL(getURL); // HttpURLConnection connection = (HttpURLConnection) url.openConnection(); // connection.connect(); // if(connection.getResponseCode() == HttpURLConnection.HTTP_OK) { // try(InputStream is = new BufferedInputStream(connection.getInputStream()); OutputStream os = new FileOutputStream(dataFile)) { // if(is.available() <= 0) { logger.error("Available <= 0"); } // byte[] databuf= new byte[1024]; // int bytesRead = is.read(databuf); // while(bytesRead > 0) { // os.write(databuf, 0, bytesRead); // bytesRead = is.read(databuf); // } // } // } // } }