Java tutorial
/** * Copyright (C) 2009 - present by OpenGamma Inc. and the OpenGamma group of companies * * Please see distribution for license. */ package com.opengamma.integration.coppclark; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.threeten.bp.LocalDate; import org.threeten.bp.format.DateTimeFormatter; import au.com.bytecode.opencsv.CSVReader; import com.opengamma.OpenGammaRuntimeException; import com.opengamma.core.holiday.HolidaySource; import com.opengamma.core.holiday.HolidayType; import com.opengamma.core.holiday.impl.NonVersionedRedisHolidaySource; import com.opengamma.core.holiday.impl.SimpleHoliday; import com.opengamma.core.id.ExternalSchemes; import com.opengamma.id.ExternalId; import com.opengamma.id.ExternalScheme; import com.opengamma.master.holiday.HolidayDocument; import com.opengamma.master.holiday.HolidayMaster; import com.opengamma.master.holiday.HolidaySearchRequest; import com.opengamma.master.holiday.HolidaySearchResult; import com.opengamma.master.holiday.ManageableHoliday; import com.opengamma.master.holiday.impl.InMemoryHolidayMaster; import com.opengamma.master.holiday.impl.MasterHolidaySource; import com.opengamma.util.ArgumentChecker; import com.opengamma.util.money.Currency; /** * Reads the holiday data from the Copp-Clark data source. * <p> * This will merge the input with the data already in the database. */ public class CoppClarkHolidayFileReader { /** * The Copp Clark scheme. */ private static final ExternalScheme COPP_CLARK_SCHEME = ExternalScheme.of("COPP_CLARK"); /** * The date format. */ private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd"); /** * An empty list of dates. */ private static final List<LocalDate> EMPTY_DATE_LIST = Collections.emptyList(); /** * The file location of the resource. */ private static final String HOLIDAY_RESOURCE_PACKAGE = "/com/coppclark/holiday/"; /** * The file location of the index file. */ private static final String HOLIDAY_INDEX_RESOURCE = HOLIDAY_RESOURCE_PACKAGE + "Index.txt"; /** * The Euro. */ private static final Currency EUR = Currency.EUR; /** * The map of Euro capitals to currencies. */ private static final Map<String, Currency> EURO_TO_OLD = new HashMap<String, Currency>(); static { EURO_TO_OLD.put("AT", Currency.of("ATS")); EURO_TO_OLD.put("BE", Currency.of("BEF")); EURO_TO_OLD.put("NL", Currency.of("NLG")); EURO_TO_OLD.put("DE", Currency.of("DEM")); EURO_TO_OLD.put("FI", Currency.of("FIM")); EURO_TO_OLD.put("FR", Currency.of("FRF")); EURO_TO_OLD.put("IE", Currency.of("IEP")); EURO_TO_OLD.put("IT", Currency.of("ITL")); EURO_TO_OLD.put("LU", Currency.of("LUF")); EURO_TO_OLD.put("MC", Currency.of("MCF")); EURO_TO_OLD.put("PT", Currency.of("PTE")); EURO_TO_OLD.put("SM", Currency.of("SML")); EURO_TO_OLD.put("ES", Currency.of("ESP")); EURO_TO_OLD.put("VA", Currency.of("VAL")); EURO_TO_OLD.put("GR", Currency.of("GRD")); EURO_TO_OLD.put("SI", Currency.of("SIT")); EURO_TO_OLD.put("CY", Currency.of("CYP")); EURO_TO_OLD.put("MT", Currency.of("MTL")); EURO_TO_OLD.put("SK", Currency.of("SKK")); EURO_TO_OLD.put("EE", Currency.of("EEK")); } /** * The streams to load the currency data. */ private final List<InputStream> _currencyStreams = new ArrayList<InputStream>(); /** * The streams to load the financial centers data. */ private final List<InputStream> _financialCentresStreams = new ArrayList<InputStream>(); /** * The streams to load the exchange settlement data. */ private final List<InputStream> _exchangeSettlementStreams = new ArrayList<InputStream>(); /** * The streams to load the exchange trading data. */ private final List<InputStream> _exchangeTradingStreams = new ArrayList<InputStream>(); /** * The holiday master to populate. */ private HolidayMaster _holidayMaster; /** * Creates a populated holiday source around the specified master. * * @param holidayMaster the holiday master to populate, not null * @return the holiday source, not null */ public static HolidaySource createPopulated(HolidayMaster holidayMaster) { CoppClarkHolidayFileReader fileReader = createPopulated0(holidayMaster); return fileReader.getHolidaySource(); } public static NonVersionedRedisHolidaySource createPopulated( NonVersionedRedisHolidaySource redisHolidaySource) { ArgumentChecker.notNull(redisHolidaySource, "redisHolidaySource"); final InMemoryHolidayMaster inMemoryHolidayMaster = new InMemoryHolidayMaster(); createPopulated(inMemoryHolidayMaster); HolidaySearchResult holidaySearchResult = inMemoryHolidayMaster.search(new HolidaySearchRequest()); List<ManageableHoliday> holidays = holidaySearchResult.getHolidays(); for (ManageableHoliday manageableHoliday : holidays) { SimpleHoliday simpleHoliday = new SimpleHoliday(); simpleHoliday.setCurrency(manageableHoliday.getCurrency()); simpleHoliday.setExchangeExternalId(manageableHoliday.getExchangeExternalId()); simpleHoliday.setHolidayDates(manageableHoliday.getHolidayDates()); simpleHoliday.setRegionExternalId(manageableHoliday.getRegionExternalId()); simpleHoliday.setType(manageableHoliday.getType()); redisHolidaySource.addHoliday(simpleHoliday); } return redisHolidaySource; } /** * Creates a populated file reader. * <p> * The values can be extracted using the methods. * * @param holidayMaster the holiday master to populate, not null * @return the holiday reader, not null */ private static CoppClarkHolidayFileReader createPopulated0(HolidayMaster holidayMaster) { CoppClarkHolidayFileReader fileReader = new CoppClarkHolidayFileReader(holidayMaster); InputStream indexStream = fileReader.getClass().getResourceAsStream(HOLIDAY_INDEX_RESOURCE); if (indexStream == null) { throw new IllegalArgumentException("Unable to find holiday index resource: " + HOLIDAY_INDEX_RESOURCE); } try { List<String> suffixes = IOUtils.readLines(indexStream, "UTF-8"); for (String suffix : suffixes) { if (StringUtils.isNotEmpty(suffix)) { String prefix = ""; if (suffix.startsWith("U")) { prefix = "Update_"; suffix = suffix.substring(1); } prefix = HOLIDAY_RESOURCE_PACKAGE + prefix; suffix += ".csv"; InputStream currencyStream = fileReader.getClass() .getResourceAsStream(prefix + "Currencies_" + suffix); if (currencyStream == null) { throw new IllegalArgumentException( "Unable to find holiday data resource: " + prefix + "Currencies_" + suffix); } fileReader.getCurrencyStreams().add(currencyStream); InputStream financialCentersStream = fileReader.getClass() .getResourceAsStream(prefix + "FinancialCentres_" + suffix); if (financialCentersStream == null) { throw new IllegalArgumentException( "Unable to find holiday data resource: " + prefix + "FinancialCentres_" + suffix); } fileReader.getFinancialCentresStreams().add(financialCentersStream); InputStream exchangeSettlementStream = fileReader.getClass() .getResourceAsStream(prefix + "ExchangeSettlement_" + suffix); if (exchangeSettlementStream == null) { throw new IllegalArgumentException( "Unable to find holiday data resource: " + prefix + "ExchangeSettlement_" + suffix); } fileReader.getExchangeSettlementStreams().add(exchangeSettlementStream); InputStream exchangeTradingStream = fileReader.getClass() .getResourceAsStream(prefix + "ExchangeTrading_" + suffix); if (exchangeTradingStream == null) { throw new IllegalArgumentException( "Unable to find holiday data resource: " + prefix + "ExchangeTrading_" + suffix); } fileReader.getExchangeTradingStreams().add(exchangeTradingStream); } } try { fileReader.read(); return fileReader; } finally { fileReader.close(); } } catch (IOException ex) { throw new OpenGammaRuntimeException("Unable to read holiday file", ex); } finally { IOUtils.closeQuietly(indexStream); } } //------------------------------------------------------------------------- /** * Creates an instance with a master to populate. * * @param holidayMaster the holiday master, not null */ public CoppClarkHolidayFileReader(HolidayMaster holidayMaster) { ArgumentChecker.notNull(holidayMaster, "holidayMaster"); _holidayMaster = holidayMaster; } //------------------------------------------------------------------------- /** * Gets the holiday master. * * @return the holiday master, not null */ public HolidayMaster getHolidayMaster() { return _holidayMaster; } /** * Gets the holiday source. * * @return the holiday source, not null */ public MasterHolidaySource getHolidaySource() { return new MasterHolidaySource(getHolidayMaster()); } /** * Gets the currency streams. * * @return the list of streams */ public List<InputStream> getCurrencyStreams() { return _currencyStreams; } /** * Gets the financial centres streams. * * @return the list of streams */ public List<InputStream> getFinancialCentresStreams() { return _financialCentresStreams; } /** * Gets the exchange settlement streams. * * @return the list of streams */ public List<InputStream> getExchangeSettlementStreams() { return _exchangeSettlementStreams; } /** * Gets the exchange trading streams. * * @return the list of streams */ public List<InputStream> getExchangeTradingStreams() { return _exchangeTradingStreams; } //------------------------------------------------------------------------- /** * Reads the streams to create the master. */ public void read() { try { parseCurrencyFile(); parseFinancialCentersFile(); parseExchangeSettlementFile(); parseExchangeTradingFile(); } catch (IOException ioe) { throw new OpenGammaRuntimeException("Problem parsing holiday data files", ioe); } } /** * Closes the streams. */ public void close() { for (InputStream stream : getCurrencyStreams()) { IOUtils.closeQuietly(stream); } for (InputStream stream : getFinancialCentresStreams()) { IOUtils.closeQuietly(stream); } for (InputStream stream : getExchangeSettlementStreams()) { IOUtils.closeQuietly(stream); } for (InputStream stream : getExchangeTradingStreams()) { IOUtils.closeQuietly(stream); } } //------------------------------------------------------------------------- private void parseCurrencyFile() throws IOException { System.out.println("Parse currencies"); Map<String, HolidayDocument> combinedMap = new HashMap<String, HolidayDocument>(512); for (InputStream stream : getCurrencyStreams()) { Map<String, HolidayDocument> fileMap = new HashMap<String, HolidayDocument>(512); @SuppressWarnings("resource") CSVReader reader = new CSVReader(new InputStreamReader(new BufferedInputStream(stream))); // header String[] row = reader.readNext(); final int ccIdx = ArrayUtils.indexOf(row, "CenterID"); final int isoCountryIdx = ArrayUtils.indexOf(row, "ISOCountryCode"); final int isoCurrencyIdx = ArrayUtils.indexOf(row, "ISOCurrencyCode"); final int eventDateIdx = ArrayUtils.indexOf(row, "EventDate"); // data while ((row = reader.readNext()) != null) { String ccId = row[ccIdx].trim(); String countryISO = row[isoCountryIdx].trim(); String currencyISO = row[isoCurrencyIdx].trim(); Currency currency = Currency.of(currencyISO); // validates format String eventDateStr = row[eventDateIdx]; LocalDate eventDate = LocalDate.parse(eventDateStr, DATE_FORMAT); HolidayDocument doc = fileMap.get(ccId); if (doc == null) { currency = fixEuro(currency, countryISO); doc = new HolidayDocument(new ManageableHoliday(currency, EMPTY_DATE_LIST)); doc.setProviderId(ExternalId.of(COPP_CLARK_SCHEME, ccId)); fileMap.put(ccId, doc); } doc.getHoliday().getHolidayDates().add(eventDate); } merge(combinedMap, fileMap); } mergeWithDatabase(combinedMap); } private Currency fixEuro(Currency currency, String countryISO) { // historic data has lots mapped to EUR, which isn't very useful if (countryISO.equals("EU") || currency.equals(EUR) == false) { return currency; } Currency newCurrency = EURO_TO_OLD.get(countryISO); if (newCurrency == null) { throw new OpenGammaRuntimeException( "EUR currency found for " + countryISO + " without mapping to true currency"); } return newCurrency; } private void parseFinancialCentersFile() throws IOException { System.out.println("Parse financial centres"); Map<String, HolidayDocument> combinedMap = new HashMap<String, HolidayDocument>(512); for (InputStream stream : getFinancialCentresStreams()) { Map<String, HolidayDocument> fileMap = new HashMap<String, HolidayDocument>(512); @SuppressWarnings("resource") CSVReader reader = new CSVReader(new InputStreamReader(new BufferedInputStream(stream))); // header String[] row = reader.readNext(); final int ccIdx = ArrayUtils.indexOf(row, "CenterID"); final int isoCountryIdx = ArrayUtils.indexOf(row, "ISOCountryCode"); final int unlocodeIdx = ArrayUtils.indexOf(row, "UN/LOCODE"); final int eventDateIdx = ArrayUtils.indexOf(row, "EventDate"); // data while ((row = reader.readNext()) != null) { String ccId = row[ccIdx].trim(); String countryISO = row[isoCountryIdx].trim(); String unlocodePart = row[unlocodeIdx].trim(); ExternalId regionId = ExternalSchemes.coppClarkRegionId(countryISO + unlocodePart); String eventDateStr = row[eventDateIdx]; LocalDate eventDate = LocalDate.parse(eventDateStr, DATE_FORMAT); HolidayDocument doc = fileMap.get(ccId); if (doc == null) { doc = new HolidayDocument(new ManageableHoliday(HolidayType.BANK, regionId, EMPTY_DATE_LIST)); doc.setProviderId(ExternalId.of(COPP_CLARK_SCHEME, ccId)); fileMap.put(ccId, doc); } doc.getHoliday().getHolidayDates().add(eventDate); } merge(combinedMap, fileMap); } mergeWithDatabase(combinedMap); } private void parseExchangeSettlementFile() throws IOException { System.out.println("Parse exchange settlements"); Map<String, HolidayDocument> combinedMap = new HashMap<String, HolidayDocument>(512); for (InputStream stream : getExchangeSettlementStreams()) { Map<String, HolidayDocument> fileMap = new HashMap<String, HolidayDocument>(512); @SuppressWarnings("resource") CSVReader reader = new CSVReader(new InputStreamReader(new BufferedInputStream(stream))); // header String[] row = reader.readNext(); final int ccIdx = ArrayUtils.indexOf(row, "CenterID"); final int isoMICCodeIdx = ArrayUtils.indexOf(row, "ISO MIC Code"); final int eventDateIdx = ArrayUtils.indexOf(row, "EventDate"); // data while ((row = reader.readNext()) != null) { String ccId = row[ccIdx].trim(); String isoMICCode = row[isoMICCodeIdx].trim(); ExternalId micId = ExternalSchemes.isoMicExchangeId(isoMICCode); String eventDateStr = row[eventDateIdx]; LocalDate eventDate = LocalDate.parse(eventDateStr, DATE_FORMAT); HolidayDocument doc = fileMap.get(ccId); if (doc == null) { doc = new HolidayDocument( new ManageableHoliday(HolidayType.SETTLEMENT, micId, EMPTY_DATE_LIST)); doc.setProviderId(ExternalId.of(COPP_CLARK_SCHEME, ccId)); fileMap.put(ccId, doc); } doc.getHoliday().getHolidayDates().add(eventDate); } merge(combinedMap, fileMap); } mergeWithDatabase(combinedMap); } private void parseExchangeTradingFile() throws IOException { System.out.println("Parse exchange trading"); Map<String, HolidayDocument> combinedMap = new HashMap<String, HolidayDocument>(512); for (InputStream stream : getExchangeTradingStreams()) { Map<String, HolidayDocument> fileMap = new HashMap<String, HolidayDocument>(512); @SuppressWarnings("resource") CSVReader reader = new CSVReader(new InputStreamReader(new BufferedInputStream(stream))); // header String[] row = reader.readNext(); final int ccIdx = ArrayUtils.indexOf(row, "CenterID"); final int isoMICCodeIdx = ArrayUtils.indexOf(row, "ISO MIC Code"); final int eventDateIdx = ArrayUtils.indexOf(row, "EventDate"); // data while ((row = reader.readNext()) != null) { String ccId = row[ccIdx].trim(); String isoMICCode = row[isoMICCodeIdx].trim(); ExternalId micId = ExternalSchemes.isoMicExchangeId(isoMICCode); String eventDateStr = row[eventDateIdx]; LocalDate eventDate = LocalDate.parse(eventDateStr, DATE_FORMAT); HolidayDocument doc = fileMap.get(ccId); if (doc == null) { doc = new HolidayDocument(new ManageableHoliday(HolidayType.TRADING, micId, EMPTY_DATE_LIST)); doc.setProviderId(ExternalId.of(COPP_CLARK_SCHEME, ccId)); fileMap.put(ccId, doc); } doc.getHoliday().getHolidayDates().add(eventDate); } merge(combinedMap, fileMap); } mergeWithDatabase(combinedMap); } //------------------------------------------------------------------------- private void merge(Map<String, HolidayDocument> combinedMap, Map<String, HolidayDocument> newMap) { for (String id : newMap.keySet()) { HolidayDocument newDoc = newMap.get(id); Collections.sort(newDoc.getHoliday().getHolidayDates()); HolidayDocument combinedDoc = combinedMap.get(id); if (combinedDoc == null) { combinedMap.put(id, newDoc); } else { mergeDates(combinedDoc, newDoc); combinedMap.put(id, newDoc); } } } private void mergeWithDatabase(Map<String, HolidayDocument> map) { Set<String> set = new HashSet<String>(); for (HolidayDocument doc : map.values()) { if (set.add(doc.getName()) == false) { System.out.println("Duplicate name: " + doc.getName()); System.exit(0); } } for (HolidayDocument doc : map.values()) { Collections.sort(doc.getHoliday().getHolidayDates()); HolidaySearchRequest search = new HolidaySearchRequest(doc.getHoliday().getType()); search.setProviderId(doc.getProviderId()); HolidaySearchResult result = _holidayMaster.search(search); if (result.getDocuments().size() == 0) { // add new data _holidayMaster.add(doc); } else if (result.getDocuments().size() == 1) { // update existing data HolidayDocument existing = result.getFirstDocument(); doc.setUniqueId(existing.getUniqueId()); doc.getHoliday().setUniqueId(existing.getUniqueId()); mergeDates(existing, doc); // only update if changed doc.setVersionFromInstant(null); doc.setVersionToInstant(null); doc.setCorrectionFromInstant(null); doc.setCorrectionToInstant(null); existing.setVersionFromInstant(null); existing.setVersionToInstant(null); existing.setCorrectionFromInstant(null); existing.setCorrectionToInstant(null); if (doc.equals(existing) == false) { // only update if changed _holidayMaster.update(doc); } } else { throw new IllegalStateException("Multiple rows in database for Copp Clark ID: " + doc.getProviderId().getValue() + " " + doc.getHoliday().getType()); } } } private void mergeDates(HolidayDocument existingDoc, HolidayDocument newDoc) { if (newDoc.getHoliday().getHolidayDates().size() == 0) { return; } // merge dates SortedSet<LocalDate> existingDates = new TreeSet<LocalDate>(existingDoc.getHoliday().getHolidayDates()); SortedSet<LocalDate> newDates = new TreeSet<LocalDate>(newDoc.getHoliday().getHolidayDates()); List<LocalDate> result = new ArrayList<LocalDate>(newDates); result.addAll(0, existingDates.headSet(newDates.first())); result.addAll(existingDates.tailSet(newDates.last().plusYears(1).withDayOfYear(1))); // file is based on whole years // store into new document newDoc.getHoliday().getHolidayDates().clear(); newDoc.getHoliday().getHolidayDates().addAll(result); } }