Java tutorial
/************************************************************************** * Copyright (C) 2010 Atlas of Living Australia * All Rights Reserved. * * The contents of this file are subject to the Mozilla Public * License Version 1.1 (the "License"); you may not use this file * except in compliance with the License. You may obtain a copy of * the License at http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or * implied. See the License for the specific language governing * rights and limitations under the License. ***************************************************************************/ package au.org.ala.biocache.web; import au.com.bytecode.opencsv.CSVReader; import au.org.ala.biocache.Config; import au.org.ala.biocache.Store; import au.org.ala.biocache.dao.SearchDAO; import au.org.ala.biocache.dto.*; import au.org.ala.biocache.dto.DownloadDetailsDTO.DownloadType; import au.org.ala.biocache.model.FullRecord; import au.org.ala.biocache.service.AuthService; import au.org.ala.biocache.service.DownloadService; import au.org.ala.biocache.service.ImageMetadataService; import au.org.ala.biocache.service.SpeciesLookupService; import au.org.ala.biocache.util.*; import org.ala.client.appender.RestLevel; import org.ala.client.model.LogEventType; import org.ala.client.model.LogEventVO; import org.ala.client.util.RestfulClient; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.support.AbstractMessageSource; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; import org.springframework.validation.BindingResult; import org.springframework.validation.ObjectError; import org.springframework.validation.Validator; import org.springframework.web.bind.WebDataBinder; import org.springframework.web.bind.annotation.*; import javax.inject.Inject; import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.validation.Valid; import java.io.*; import java.util.*; import java.util.regex.Pattern; /** * Occurrences controller for the biocache web services. * * @author "Nick dos Remedios <Nick.dosRemedios@csiro.au>" * @author "Natasha Carter <Natasha.Carter@csiro.au>" */ @Controller public class OccurrenceController extends AbstractSecureController { /** Logger initialisation */ private final static Logger logger = Logger.getLogger(OccurrenceController.class); /** Fulltext search DAO */ @Inject protected SearchDAO searchDAO; /** Data Resource DAO */ @Inject protected SearchUtils searchUtils; @Inject protected RestfulClient restfulClient; @Inject protected SpeciesLookupService speciesLookupService; @Inject protected AuthService authService; @Inject protected ContactUtils contactUtils; @Inject protected AssertionUtils assertionUtils; @Inject protected DownloadService downloadService; @Inject private AbstractMessageSource messageSource; @Inject private ImageMetadataService imageMetadataService; @Autowired private Validator validator; /** Name of view for site home page */ private String HOME = "homePage"; private String VALIDATION_ERROR = "error/validationError"; @Value("${webservices.root:http://localhost:8080/biocache-service}") protected String hostUrl; /** The response to be returned for the isAustralian test */ @Value("${taxon.id.pattern:urn:lsid:biodiversity.org.au[a-zA-Z0-9\\.:-]*}") protected String taxonIDPatternString; @Value("${native.country:Australia}") protected String nativeCountry; /** Compiled pattern for taxon IDs */ protected Pattern taxonIDPattern; @Value("${media.url:http://biocache.ala.org.au/biocache-media/}") protected String biocacheMediaUrl; @Value("${facet.config:/data/biocache/config/facets.json}") protected String facetConfig; public Pattern getTaxonIDPattern() { if (taxonIDPattern == null) { taxonIDPattern = Pattern.compile(taxonIDPatternString); } return taxonIDPattern; } /** * Need to initialise the validator to be used otherwise the @Valid annotation will not work * @param binder */ @InitBinder protected void initBinder(WebDataBinder binder) { binder.setValidator(validator); } /** * Custom handler for the welcome view. * <p> * Note that this handler relies on the RequestToViewNameTranslator to * determine the logical view name based on the request URL: "/welcome.do" * -> "welcome". * * @return viewname to render */ @RequestMapping("/") public String homePageHandler(Model model) { model.addAttribute("webservicesRoot", hostUrl); ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); InputStream input = classLoader.getResourceAsStream("/git.properties"); if (input != null) { try { Properties versionProperties = new Properties(); versionProperties.load(input); model.addAttribute("versionInfo", versionProperties); StringBuffer sb = new StringBuffer(); for (String name : versionProperties.stringPropertyNames()) { sb.append(name + " : " + versionProperties.getProperty(name) + "\n"); } model.addAttribute("versionInfoString", sb.toString()); } catch (Exception e) { logger.error(e.getMessage(), e); } } return HOME; } /** * Custom handler for the welcome view. * <p> * Note that this handler relies on the RequestToViewNameTranslator to * determine the logical view name based on the request URL: "/welcome.do" * -> "welcome". * * @return viewname to render */ @RequestMapping("/oldapi") public String oldApiHandler(Model model) { model.addAttribute("webservicesRoot", hostUrl); return "oldapi"; } @RequestMapping("/active/download/stats") public @ResponseBody List<DownloadDetailsDTO> getCurrentDownloads() { return downloadService.getCurrentDownloads(); } /** * Returns the default facets that are applied to a search * @return */ @RequestMapping("/search/facets") public @ResponseBody String[] listAllFacets() { return new SearchRequestParams().getFacets(); } /** * Returns the default facets grouped by themes that are applied to a search * @return */ @RequestMapping("/search/grouped/facets") public @ResponseBody List groupFacets() throws IOException { return new FacetThemes(facetConfig).allThemes; } /** * Returns the content of the messages.properties file. * Can also return language specific versions, such as * messages_fr.properties if requested via qualifier @PathVariable. * * @param response * @throws Exception */ @RequestMapping("/facets/i18n{qualifier:.*}*") public void writei18nPropertiesFile(@PathVariable("qualifier") String qualifier, HttpServletRequest request, HttpServletResponse response) throws Exception { qualifier = (StringUtils.isNotEmpty(qualifier)) ? qualifier : ".properties"; logger.debug("qualifier = " + qualifier); InputStream is = request.getSession().getServletContext() .getResourceAsStream("/WEB-INF/messages" + qualifier); OutputStream os = response.getOutputStream(); if (is != null) { byte[] buffer = new byte[1024]; int bytesRead; while ((bytesRead = is.read(buffer)) != -1) { os.write(buffer, 0, bytesRead); } } os.flush(); os.close(); } /** * Returns a list with the details of the index field * @return * @throws Exception */ @RequestMapping("index/fields") public @ResponseBody Set<IndexFieldDTO> getIndexedFields( @RequestParam(value = "fl", required = false) String fields) throws Exception { if (fields == null) return searchDAO.getIndexedFields(); else return searchDAO.getIndexFieldDetails(fields.split(",")); } /** * Returns a facet list including the number of distinct values for a field * @param requestParams * @return * @throws Exception */ @RequestMapping("occurrence/facets") public @ResponseBody List<FacetResultDTO> getOccurrenceFacetDetails(SpatialSearchRequestParams requestParams) throws Exception { return searchDAO.getFacetCounts(requestParams); } /** * Returns a list of image urls for the supplied taxon guid. * An empty list is returned when no images are available. * * @param guid * @return * @throws Exception */ @RequestMapping(value = { "/images/taxon/{guid:.+}.json*", "/images/taxon/{guid:.+}*" }) public @ResponseBody List<String> getImages(@PathVariable("guid") String guid) throws Exception { SpatialSearchRequestParams srp = new SpatialSearchRequestParams(); srp.setQ("lsid:" + guid); srp.setPageSize(0); srp.setFacets(new String[] { "image_url" }); SearchResultDTO results = searchDAO.findByFulltextSpatialQuery(srp, null); if (results.getFacetResults().size() > 0) { List<FieldResultDTO> fieldResults = results.getFacetResults().iterator().next().getFieldResult(); ArrayList<String> images = new ArrayList<String>(fieldResults.size()); for (FieldResultDTO fr : fieldResults) images.add(fr.getLabel()); return images; } return Collections.EMPTY_LIST; } /** * Checks to see if the supplied GUID represents an Australian species. * @param guid * @return * @throws Exception */ @RequestMapping(value = { "/australian/taxon/{guid:.+}.json*", "/australian/taxon/{guid:.+}*", "/native/taxon/{guid:.+}.json*", "/native/taxon/{guid:.+}*" }) public @ResponseBody NativeDTO isAustralian(@PathVariable("guid") String guid) throws Exception { //check to see if we have any occurrences on Australia country:Australia or state != empty SpatialSearchRequestParams requestParams = new SpatialSearchRequestParams(); requestParams.setPageSize(0); requestParams.setFacets(new String[] {}); String query = "lsid:" + guid + " AND " + "(country:\"" + nativeCountry + "\" OR state:[* TO *]) AND geospatial_kosher:true"; requestParams.setQ(query); NativeDTO adto = new NativeDTO(); adto.setTaxonGuid(guid); SearchResultDTO results = searchDAO.findByFulltextSpatialQuery(requestParams, null); adto.setHasOccurrenceRecords(results.getTotalRecords() > 0); adto.setIsNSL(getTaxonIDPattern().matcher(guid).matches()); if (adto.isHasOccurrences()) { //check to see if the records have only been provided by citizen science //TODO change this to a confidence setting after it has been included in the index requestParams.setQ("lsid:" + guid + " AND (provenance:\"Published dataset\")"); results = searchDAO.findByFulltextSpatialQuery(requestParams, null); adto.setHasCSOnly(results.getTotalRecords() == 0); } return adto; } /** * Returns the complete list of Occurrences */ @RequestMapping(value = { "/occurrences", "/occurrences/collections", "/occurrences/institutions", "/occurrences/dataResources", "/occurrences/dataProviders", "/occurrences/taxa", "/occurrences/dataHubs" }, method = RequestMethod.GET) public @ResponseBody SearchResultDTO listOccurrences(Model model) throws Exception { SpatialSearchRequestParams srp = new SpatialSearchRequestParams(); srp.setQ("*:*"); return occurrenceSearch(srp); } /** * Occurrence search page uses SOLR JSON to display results * * @param model * @return * @throws Exception */ @RequestMapping(value = { "/occurrences/taxon/{guid:.+}.json*", "/occurrences/taxon/{guid:.+}*", "/occurrences/taxa/{guid:.+}*" }, method = RequestMethod.GET) public @ResponseBody SearchResultDTO occurrenceSearchByTaxon(SpatialSearchRequestParams requestParams, @PathVariable("guid") String guid) throws Exception { requestParams.setQ("lsid:" + guid); SearchUtils.setDefaultParams(requestParams); return occurrenceSearch(requestParams); } /** * Obtains a list of the sources for the supplied guid. * * I don't think that this should be necessary. We should be able to * configure the requestParams facets to contain the collectino_uid, institution_uid * data_resource_uid and data_provider_uid * * It also handle's the logging for the BIE. * //TODO Work out what to do with this * @param guid * @throws Exception */ @RequestMapping(value = "/occurrences/taxon/source/{guid:.+}.json*", method = RequestMethod.GET) public @ResponseBody List<OccurrenceSourceDTO> sourceByTaxon(SpatialSearchRequestParams requestParams, @PathVariable("guid") String guid) throws Exception { requestParams.setQ("lsid:" + guid); Map<String, Integer> sources = searchDAO.getSourcesForQuery(requestParams); //now turn them to a list of OccurrenceSourceDTO return searchUtils.getSourceInformation(sources); } /** * Occurrence search for a given collection, institution, data_resource or data_provider. * * @param requestParams The search parameters * @param uid The uid for collection, institution, data_resource or data_provider * @param model * @return * @throws Exception */ @RequestMapping(value = { "/occurrences/collections/{uid}", "/occurrences/institutions/{uid}", "/occurrences/dataResources/{uid}", "/occurrences/dataProviders/{uid}", "/occurrences/dataHubs/{uid}" }, method = RequestMethod.GET) public @ResponseBody SearchResultDTO occurrenceSearchForUID(SpatialSearchRequestParams requestParams, @PathVariable("uid") String uid, Model model) throws Exception { SearchResultDTO searchResult = new SearchResultDTO(); // no query so exit method if (StringUtils.isEmpty(uid)) { return searchResult; } SearchUtils.setDefaultParams(requestParams); //update the request params so the search caters for the supplied uid searchUtils.updateCollectionSearchString(requestParams, uid); logger.debug("solr query: " + requestParams); return occurrenceSearch(requestParams); } /** * Spatial search for either a taxon name or full text text search * @param model * @deprecated use {@link #occurrenceSearch(SpatialSearchRequestParams)} * @return * @throws Exception */ @RequestMapping(value = "/occurrences/searchByArea*", method = RequestMethod.GET) @Deprecated public @ResponseBody SearchResultDTO occurrenceSearchByArea(SpatialSearchRequestParams requestParams, Model model) throws Exception { SearchResultDTO searchResult = new SearchResultDTO(); if (StringUtils.isEmpty(requestParams.getQ())) { return searchResult; } //searchUtils.updateSpatial(requestParams); searchResult = searchDAO.findByFulltextSpatialQuery(requestParams, null); model.addAttribute("searchResult", searchResult); if (logger.isDebugEnabled()) { logger.debug("Returning results set with: " + searchResult.getTotalRecords()); } return searchResult; } private SearchResultDTO occurrenceSearch(SpatialSearchRequestParams requestParams) throws Exception { return occurrenceSearch(requestParams, null, false, null, null); } /** * Occurrence search page uses SOLR JSON to display results * * @return * @throws Exception */ @RequestMapping(value = { "/occurrences/search.json*", "/occurrences/search*" }, method = RequestMethod.GET) public @ResponseBody SearchResultDTO occurrenceSearch(SpatialSearchRequestParams requestParams, @RequestParam(value = "apiKey", required = false) String apiKey, @RequestParam(value = "im", required = false, defaultValue = "false") Boolean lookupImageMetadata, HttpServletRequest request, HttpServletResponse response) throws Exception { // handle empty param values, e.g. &sort=&dir= SearchUtils.setDefaultParams(requestParams); Map<String, String[]> map = request != null ? SearchUtils.getExtraParams(request.getParameterMap()) : null; if (map != null) { map.remove("apiKey"); } logger.debug("occurrence search params = " + requestParams); SearchResultDTO srtdto = null; if (apiKey == null) { srtdto = searchDAO.findByFulltextSpatialQuery(requestParams, map); } else { srtdto = occurrenceSearchSensitive(requestParams, apiKey, request, response); } if (lookupImageMetadata) { //use the image service API //grab the list of IDs List<String> occurrenceIDs = new ArrayList<String>(); for (OccurrenceIndex oi : srtdto.getOccurrences()) { occurrenceIDs.add(oi.getUuid()); } Map<String, List<Map<String, Object>>> imageMap = imageMetadataService .getImageMetadataForOccurrences(occurrenceIDs); for (OccurrenceIndex oi : srtdto.getOccurrences()) { //lookup metadata List<Map<String, Object>> imageMetadata = imageMap.get(oi.getUuid()); oi.setImageMetadata(imageMetadata); } } return srtdto; } public @ResponseBody SearchResultDTO occurrenceSearchSensitive(SpatialSearchRequestParams requestParams, @RequestParam(value = "apiKey", required = true) String apiKey, HttpServletRequest request, HttpServletResponse response) throws Exception { // handle empty param values, e.g. &sort=&dir= if (shouldPerformOperation(apiKey, response, false)) { SearchUtils.setDefaultParams(requestParams); Map<String, String[]> map = SearchUtils.getExtraParams(request.getParameterMap()); if (map != null) { map.remove("apiKey"); } logger.debug("occurrence search params = " + requestParams); SearchResultDTO searchResult = searchDAO.findByFulltextSpatialQuery(requestParams, true, map); return searchResult; } return null; } /** * Occurrence search page uses SOLR JSON to display results * * @return * @throws Exception */ @RequestMapping(value = { "/cache/refresh" }, method = RequestMethod.GET) public @ResponseBody String refreshCache() throws Exception { searchDAO.refreshCaches(); return null; } /** * Downloads the complete list of values in the supplied facet * * ONLY 1 facet should be included in the params. * * @param requestParams * @param response * @throws Exception */ @RequestMapping(value = "/occurrences/facets/download*", method = RequestMethod.GET) public void downloadFacet(DownloadRequestParams requestParams, @RequestParam(value = "count", required = false, defaultValue = "false") boolean includeCount, @RequestParam(value = "lookup", required = false, defaultValue = "false") boolean lookupName, @RequestParam(value = "synonym", required = false, defaultValue = "false") boolean includeSynonyms, @RequestParam(value = "ip", required = false) String ip, HttpServletRequest request, HttpServletResponse response) throws Exception { if (requestParams.getFacets().length > 0) { ip = ip == null ? getIPAddress(request) : ip; DownloadDetailsDTO dd = downloadService.registerDownload(requestParams, ip, DownloadDetailsDTO.DownloadType.FACET); try { String filename = requestParams.getFile() != null ? requestParams.getFile() : requestParams.getFacets()[0]; response.setHeader("Cache-Control", "must-revalidate"); response.setHeader("Pragma", "must-revalidate"); response.setHeader("Content-Disposition", "attachment;filename=" + filename + ".csv"); response.setContentType("text/csv"); searchDAO.writeFacetToStream(requestParams, includeCount, lookupName, includeSynonyms, response.getOutputStream(), dd); } finally { downloadService.unregisterDownload(dd); } } } /** * Webservice to support bulk downloads for a long list of queries for a single field. * NOTE: triggered on "Download Records" button * * @param response * @param request * @param separator * @return * @throws Exception */ @RequestMapping(value = "/occurrences/batchSearch", method = RequestMethod.POST, params = "action=Download") public void batchDownload(HttpServletResponse response, HttpServletRequest request, @RequestParam(value = "queries", required = true, defaultValue = "") String queries, @RequestParam(value = "field", required = true, defaultValue = "") String field, @RequestParam(value = "separator", defaultValue = "\n") String separator, @RequestParam(value = "title", required = false) String title) throws Exception { logger.info("/occurrences/batchSearch with action=Download Records"); Long qid = getQidForBatchSearch(queries, field, separator, title); if (qid != null) { String webservicesRoot = request.getSession().getServletContext().getInitParameter("webservicesRoot"); response.sendRedirect(webservicesRoot + "/occurrences/download?q=qid:" + qid); } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST); } } @RequestMapping(value = "/occurrences/download/batchFile", method = RequestMethod.GET) public String batchDownload(HttpServletRequest request, @Valid final DownloadRequestParams params, BindingResult result, @RequestParam(value = "file", required = true) String filepath, @RequestParam(value = "directory", required = true, defaultValue = "/data/biocache-exports") final String directory, @RequestParam(value = "ip", required = false) String ip, Model model) throws Exception { if (result.hasErrors()) { logger.info("validation failed " + result.getErrorCount() + " checks"); logger.debug(result.toString()); model.addAttribute("errorMessage", getValidationErrorMessage(result)); //response.setStatus(response.SC_INTERNAL_SERVER_ERROR); return VALIDATION_ERROR;//result.toString(); } final File file = new File(filepath); final SpeciesLookupService mySpeciesLookupService = this.speciesLookupService; ip = ip == null ? getIPAddress(request) : ip; final DownloadDetailsDTO dd = downloadService.registerDownload(params, ip, DownloadType.RECORDS_INDEX); if (file.exists()) { Thread t = new Thread() { @Override public void run() { try { //start a thread CSVReader reader = new CSVReader(new FileReader(file)); String[] row = reader.readNext(); while (row != null) { //get an lsid for the name String lsid = mySpeciesLookupService.getGuidForName(row[0]); if (lsid != null) { try { //download records for this row String outputFilePath = directory + File.separatorChar + row[0].replace(" ", "_") + ".txt"; String citationFilePath = directory + File.separatorChar + row[0].replace(" ", "_") + "_citations.txt"; logger.debug( "Outputting results to:" + outputFilePath + ", with LSID: " + lsid); FileOutputStream output = new FileOutputStream(outputFilePath); params.setQ("lsid:\"" + lsid + "\""); Map<String, Integer> uidStats = searchDAO.writeResultsFromIndexToStream(params, output, false, dd, false); FileOutputStream citationOutput = new FileOutputStream(citationFilePath); downloadService.getCitations(uidStats, citationOutput, params.getSep(), params.getEsc()); citationOutput.flush(); citationOutput.close(); output.flush(); output.close(); } catch (Exception e) { logger.error(e.getMessage(), e); } } else { logger.error("Unable to match name: " + row[0]); } row = reader.readNext(); } } catch (Exception e) { logger.error(e.getMessage(), e); } finally { downloadService.unregisterDownload(dd); } } }; t.start(); } return null; } /** * Given a list of queries for a single field, return an AJAX response with the qid (cached query id) * NOTE: triggered on "Search" button * * @param response * @param queries * @param separator * @return * @throws Exception */ @RequestMapping(value = "/occurrences/batchSearch", method = RequestMethod.POST, params = "action=Search") public void batchSearch(HttpServletResponse response, @RequestParam(value = "redirectBase", required = true, defaultValue = "") String redirectBase, @RequestParam(value = "queries", required = true, defaultValue = "") String queries, @RequestParam(value = "field", required = true, defaultValue = "") String field, @RequestParam(value = "separator", defaultValue = "\n") String separator, @RequestParam(value = "title", required = false) String title) throws Exception { logger.info("/occurrences/batchSearch with action=Search"); Long qid = getQidForBatchSearch(queries, field, separator, title); if (qid != null && StringUtils.isNotBlank(redirectBase)) { response.sendRedirect(redirectBase + "?q=qid:" + qid); } else { response.sendError(HttpServletResponse.SC_BAD_REQUEST, ""); } } /** * Common method for getting a QID for a batch field query * * @param listOfNames * @param separator * @return * @throws IOException * @throws ParamsCacheSizeException */ private Long getQidForBatchSearch(String listOfNames, String field, String separator, String title) throws IOException, ParamsCacheSizeException { String[] rawParts = listOfNames.split(separator); List<String> parts = new ArrayList<String>(); for (String part : rawParts) { String normalised = StringUtils.trimToNull(part); if (normalised != null) { parts.add(field + ":\"" + normalised + "\""); } } if (parts.isEmpty()) { return null; } String q = StringUtils.join(parts.toArray(new String[0]), " OR "); title = title == null ? q : title; long qid = ParamsCache.put(q, title, null, null, null); logger.info("batchSearch: qid = " + qid); return qid; } /** * Webservice to report the occurrence counts for the supplied list of taxa * */ @RequestMapping(value = "/occurrences/taxaCount", method = { RequestMethod.POST, RequestMethod.GET }) public @ResponseBody Map<String, Integer> occurrenceSpeciesCounts(HttpServletResponse response, HttpServletRequest request, @RequestParam(defaultValue = "\n") String separator) throws Exception { String listOfGuids = (String) request.getParameter("guids"); String[] rawGuids = listOfGuids.split(separator); List<String> guids = new ArrayList<String>(); for (String guid : rawGuids) { String normalised = StringUtils.trimToNull(guid); if (normalised != null) guids.add(normalised); } return searchDAO.getOccurrenceCountsForTaxa(guids); } /** * Occurrence search page uses SOLR JSON to display results * * Please NOTE that the q and fq provided to this URL should be obtained * from SearchResultDTO.urlParameters * * @return * @throws Exception */ @RequestMapping(value = "/occurrences/download*", method = RequestMethod.GET) public String occurrenceDownload(@Valid DownloadRequestParams requestParams, BindingResult result, @RequestParam(value = "ip", required = false) String ip, @RequestParam(value = "apiKey", required = false) String apiKey, Model model, HttpServletResponse response, HttpServletRequest request) throws Exception { //org.springframework.validation.BindException errors = new org.springframework.validation.BindException(requestParams,"requestParams"); //validator.validate(requestParams, errors); //check to see if the DownloadRequestParams are valid if (result.hasErrors()) { logger.info("validation failed " + result.getErrorCount() + " checks"); logger.debug(result.toString()); model.addAttribute("errorMessage", getValidationErrorMessage(result)); //response.setStatus(response.SC_INTERNAL_SERVER_ERROR); return VALIDATION_ERROR;//result.toString(); } ip = ip == null ? getIPAddress(request) : ip;//request.getRemoteAddr():ip; ServletOutputStream out = response.getOutputStream(); //search params must have a query or formatted query for the downlaod to work if (requestParams.getQ().isEmpty() && requestParams.getFormattedQuery().isEmpty()) { return null; } if (apiKey != null) { return occurrenceSensitiveDownload(requestParams, apiKey, ip, false, response, request); } try { downloadService.writeQueryToStream(requestParams, response, ip, out, false, false); } catch (Exception e) { logger.error(e.getMessage(), e); } return null; } @RequestMapping(value = "/occurrences/index/download*", method = RequestMethod.GET) public String occurrenceIndexDownload(@Valid DownloadRequestParams requestParams, BindingResult result, @RequestParam(value = "apiKey", required = false) String apiKey, @RequestParam(value = "ip", required = false) String ip, Model model, HttpServletResponse response, HttpServletRequest request) throws Exception { if (result.hasErrors()) { logger.info("validation failed " + result.getErrorCount() + " checks"); logger.debug(result.toString()); model.addAttribute("errorMessage", getValidationErrorMessage(result)); //response.setStatus(response.SC_INTERNAL_SERVER_ERROR); return VALIDATION_ERROR;//result.toString(); } ip = ip == null ? getIPAddress(request) : ip; ServletOutputStream out = response.getOutputStream(); //search params must have a query or formatted query for the download to work if (requestParams.getQ().isEmpty() && requestParams.getFormattedQuery().isEmpty()) { return null; } if (apiKey != null) { occurrenceSensitiveDownload(requestParams, apiKey, ip, true, response, request); return null; } try { downloadService.writeQueryToStream(requestParams, response, ip, out, false, true); } catch (Exception e) { logger.error(e.getMessage(), e); } return null; } //@RequestMapping(value = "/sensitive/occurrences/download*", method = RequestMethod.GET) public String occurrenceSensitiveDownload(DownloadRequestParams requestParams, String apiKey, String ip, boolean fromIndex, HttpServletResponse response, HttpServletRequest request) throws Exception { if (shouldPerformOperation(apiKey, response, false)) { ip = ip == null ? getIPAddress(request) : ip; ServletOutputStream out = response.getOutputStream(); //search params must have a query or formatted query for the downlaod to work if (requestParams.getQ().isEmpty() && requestParams.getFormattedQuery().isEmpty()) { return null; } downloadService.writeQueryToStream(requestParams, response, ip, out, true, fromIndex); } return null; } /** * Returns the IP address for the supplied request. It will look for the existence of * an X-Forwarded-For Header before extracting it from the request. * @param request * @return IP Address of the request */ private String getIPAddress(HttpServletRequest request) { //check to see if proxied. String forwardedFor = request.getHeader("X-Forwarded-For"); return forwardedFor == null ? request.getRemoteAddr() : forwardedFor; } /** * Utility method for retrieving a list of occurrences. Mainly added to help debug * web services for that a developer can retrieve example UUIDs. * * @throws Exception */ @RequestMapping(value = { "/occurrences/nearest" }, method = RequestMethod.GET) public @ResponseBody Map<String, Object> nearestOccurrence(SpatialSearchRequestParams requestParams) throws Exception { logger.debug(String.format("Received lat: %f, lon:%f, radius:%f", requestParams.getLat(), requestParams.getLon(), requestParams.getRadius())); if (requestParams.getLat() == null || requestParams.getLon() == null) { return new HashMap<String, Object>(); } //requestParams.setRadius(1f); requestParams.setDir("asc"); requestParams.setFacet(false); SearchResultDTO searchResult = searchDAO.findByFulltextSpatialQuery(requestParams, null); List<OccurrenceIndex> ocs = searchResult.getOccurrences(); if (!ocs.isEmpty()) { Map<String, Object> results = new HashMap<String, Object>(); OccurrenceIndex oc = ocs.get(0); Double decimalLatitude = oc.getDecimalLatitude(); Double decimalLongitude = oc.getDecimalLongitude(); Double distance = distInMetres(requestParams.getLat().doubleValue(), requestParams.getLon().doubleValue(), decimalLatitude, decimalLongitude); results.put("distanceInMeters", distance); results.put("occurrence", oc); return results; } else { return new HashMap<String, Object>(); } } private Double distInMetres(Double lat1, Double lon1, Double lat2, Double lon2) { Double R = 6371000d; // km Double dLat = Math.toRadians(lat2 - lat1); Double dLon = Math.toRadians(lon2 - lon1); Double lat1Rad = Math.toRadians(lat1); Double lat2Rad = Math.toRadians(lat2); Double a = Math.sin(dLat / 2) * Math.sin(dLat / 2) + Math.sin(dLon / 2) * Math.sin(dLon / 2) * Math.cos(lat1Rad) * Math.cos(lat2Rad); Double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); return R * c; } /** * Dumps the distinct latitudes and longitudes that are used in the * connected index (to 4 decimal places) */ @RequestMapping(value = "/occurrences/coordinates*") public void dumpDistinctLatLongs(SearchRequestParams requestParams, HttpServletResponse response) throws Exception { requestParams.setFacets(new String[] { "lat_long" }); if (requestParams.getQ().length() < 1) requestParams.setQ("*:*"); ServletOutputStream out = response.getOutputStream(); searchDAO.writeCoordinatesToStream(requestParams, out); } /** * Occurrence record page * * When user supplies a uuid that is not found search for a unique record * with the supplied occurrenc_id * * Returns a SearchResultDTO when there is more than 1 record with the supplied UUID * * TODO move to service layer * * @param uuid * @throws Exception */ @RequestMapping(value = { "/occurrence/compare/{uuid}.json", "/occurrence/compare/{uuid}" }, method = RequestMethod.GET) public @ResponseBody Object showOccurrence(@PathVariable("uuid") String uuid) { Map values = Store.getComparisonByUuid(uuid); if (values.isEmpty()) values = Store.getComparisonByRowKey(uuid); //substitute the values for recordedBy if it is an authenticated user if (values.containsKey("Occurrence")) { //String recordedBy = values.get("recordedBy").toString(); List<au.org.ala.biocache.parser.ProcessedValue> compareList = (List<au.org.ala.biocache.parser.ProcessedValue>) values .get("Occurrence"); List<au.org.ala.biocache.parser.ProcessedValue> newList = new ArrayList<au.org.ala.biocache.parser.ProcessedValue>(); for (au.org.ala.biocache.parser.ProcessedValue pv : compareList) { if (pv.getName().equals("recordedBy")) { logger.info(pv); String raw = authService.substituteEmailAddress(pv.getRaw()); String processed = authService.substituteEmailAddress(pv.getProcessed()); au.org.ala.biocache.parser.ProcessedValue newpv = new au.org.ala.biocache.parser.ProcessedValue( "recordedBy", raw, processed); newList.add(newpv); } else { newList.add(pv); } } values.put("Occurrence", newList); } return values; } /** * Returns a comparison of the occurrence versions. * @param uuid * @return */ @RequestMapping(value = { "/occurrence/compare*" }, method = RequestMethod.GET) public @ResponseBody Object compareOccurrenceVersions( @RequestParam(value = "uuid", required = true) String uuid) { return showOccurrence(uuid); } /** * Returns the records uuids that have been deleted since the fromDate inclusive. * * @param fromDate * @param response * @return * @throws Exception */ @RequestMapping(value = { "/occurrence/deleted" }, method = RequestMethod.GET) public @ResponseBody String[] getDeleteOccurrences( @RequestParam(value = "date", required = true) String fromDate, HttpServletResponse response) throws Exception { String[] deletedRecords = new String[0]; try { //date must be in a yyyy-MM-dd format Date date = org.apache.commons.lang.time.DateUtils.parseDate(fromDate, new String[] { "yyyy-MM-dd" }); deletedRecords = Store.getDeletedRecords(date); if (deletedRecords == null) { deletedRecords = new String[0]; } } catch (java.text.ParseException e) { response.sendError(HttpServletResponse.SC_BAD_REQUEST, "Invalid date format. Please provide date as yyyy-MM-dd."); } catch (Exception e) { logger.error(e.getMessage(), e); response.sendError(500, "Problem retrieving details of deleted records."); } return deletedRecords; } /** * Occurrence record page * * When user supplies a uuid that is not found search for a unique record * with the supplied occurrence_id * * Returns a SearchResultDTO when there is more than 1 record with the supplied UUID * * @param uuid * @param apiKey * @throws Exception */ @RequestMapping(value = { "/occurrence/{uuid:.+}", "/occurrences/{uuid:.+}", "/occurrence/{uuid:.+}.json", "/occurrences/{uuid:.+}.json" }, method = RequestMethod.GET) public @ResponseBody Object showOccurrence(@PathVariable("uuid") String uuid, @RequestParam(value = "apiKey", required = false) String apiKey, @RequestParam(value = "ip", required = false) String ip, HttpServletRequest request, HttpServletResponse response) throws Exception { ip = ip == null ? getIPAddress(request) : ip; if (apiKey != null) { return showSensitiveOccurrence(uuid, apiKey, ip, request, response); } return getOccurrenceInformation(uuid, ip, request, false); } @RequestMapping(value = { "/sensitive/occurrence/{uuid:.+}", "/sensitive/occurrences/{uuid:.+}", "/sensitive/occurrence/{uuid:.+}.json", "/senstive/occurrences/{uuid:.+}.json" }, method = RequestMethod.GET) public @ResponseBody Object showSensitiveOccurrence(@PathVariable("uuid") String uuid, @RequestParam(value = "apiKey", required = true) String apiKey, @RequestParam(value = "ip", required = false) String ip, HttpServletRequest request, HttpServletResponse response) throws Exception { ip = ip == null ? getIPAddress(request) : ip; if (shouldPerformOperation(apiKey, response)) { return getOccurrenceInformation(uuid, ip, request, true); } return null; } private Object getOccurrenceInformation(String uuid, String ip, HttpServletRequest request, boolean includeSensitive) throws Exception { logger.debug("Retrieving occurrence record with guid: '" + uuid + "'"); FullRecord[] fullRecord = Store.getAllVersionsByUuid(uuid, includeSensitive); if (fullRecord == null) { //get the rowKey for the supplied uuid in the index //This is a workaround. There seems to be an issue on Cassandra with retrieving uuids that start with e or f SpatialSearchRequestParams srp = new SpatialSearchRequestParams(); srp.setQ("id:" + uuid); srp.setPageSize(1); srp.setFacets(new String[] {}); SearchResultDTO results = occurrenceSearch(srp); if (results.getTotalRecords() > 0) fullRecord = Store.getAllVersionsByRowKey(results.getOccurrences().get(0).getRowKey(), includeSensitive); } if (fullRecord == null) { //check to see if we have an occurrence id SpatialSearchRequestParams srp = new SpatialSearchRequestParams(); srp.setQ("occurrence_id:" + uuid); SearchResultDTO result = occurrenceSearch(srp); if (result.getTotalRecords() > 1) return result; else if (result.getTotalRecords() == 0) return new OccurrenceDTO(); else fullRecord = Store.getAllVersionsByUuid(result.getOccurrences().get(0).getUuid(), includeSensitive); } OccurrenceDTO occ = new OccurrenceDTO(fullRecord); // now update the values required for the authService if (fullRecord != null) { //TODO - move this logic to service layer //raw record may need recordedBy to be changed //NC 2013-06-26: The substitution was removed in favour of email obscuring due to numeric id's being used for non-ALA data resources fullRecord[0].getOccurrence().setRecordedBy( authService.substituteEmailAddress(fullRecord[0].getOccurrence().getRecordedBy())); //processed record may need recordedBy modified in case it was an email address. fullRecord[1].getOccurrence().setRecordedBy( authService.substituteEmailAddress(fullRecord[1].getOccurrence().getRecordedBy())); //hide the email addresses in the raw miscProperties Map<String, String> miscProps = fullRecord[0].miscProperties(); for (Map.Entry<String, String> entry : miscProps.entrySet()) { if (entry.getValue().contains("@")) entry.setValue(authService.substituteEmailAddress(entry.getValue())); } //if the raw record contains a userId we will need to include the alaUserName in the DTO if (fullRecord[0].getOccurrence().getUserId() != null) { occ.setAlaUserName(authService.getDisplayNameFor(fullRecord[0].getOccurrence().getUserId())); } else if (fullRecord[1].getOccurrence().getUserId() != null) { occ.setAlaUserName(authService.getDisplayNameFor(fullRecord[1].getOccurrence().getUserId())); } } String rowKey = occ.getProcessed().getRowKey(); //assertions are based on the row key not uuid occ.setSystemAssertions(Store.getAllSystemAssertions(rowKey)); occ.setUserAssertions(assertionUtils.getUserAssertions(occ)); //retrieve details of the media files List<MediaDTO> soundDtos = getSoundDtos(occ); if (!soundDtos.isEmpty()) { occ.setSounds(soundDtos); } //ADD THE DIFFERENT IMAGE FORMATS...thumb,small,large,raw setupImageUrls(occ); //fix media store URLs Config.mediaStore().convertPathsToUrls(occ.getRaw(), biocacheMediaUrl); Config.mediaStore().convertPathsToUrls(occ.getProcessed(), biocacheMediaUrl); //log the statistics for viewing the record logViewEvent(ip, occ, null, "Viewing Occurrence Record " + uuid); return occ; } private void logViewEvent(String ip, OccurrenceDTO occ, String email, String reason) { //String ip = request.getLocalAddr(); Map<String, Integer> uidStats = new HashMap<String, Integer>(); if (occ.getProcessed() != null && occ.getProcessed().getAttribution() != null) { if (occ.getProcessed().getAttribution().getCollectionUid() != null) { uidStats.put(occ.getProcessed().getAttribution().getCollectionUid(), 1); } if (occ.getProcessed().getAttribution().getInstitutionUid() != null) { uidStats.put(occ.getProcessed().getAttribution().getInstitutionUid(), 1); } if (occ.getProcessed().getAttribution().getDataProviderUid() != null) uidStats.put(occ.getProcessed().getAttribution().getDataProviderUid(), 1); if (occ.getProcessed().getAttribution().getDataResourceUid() != null) uidStats.put(occ.getProcessed().getAttribution().getDataResourceUid(), 1); } LogEventVO vo = new LogEventVO(LogEventType.OCCURRENCE_RECORDS_VIEWED, email, reason, ip, uidStats); logger.log(RestLevel.REMOTE, vo); } /** * Constructs an error message to be displayed. The error message is based on validation checks that * were performed and stored in the supplied result. * * TODO: If we decide to perform more detailed validations elsewhere it maybe worth providing this in a * util or service class. * * @param result The result from the validation. * @return A string representation that can be displayed in a browser. */ private String getValidationErrorMessage(BindingResult result) { StringBuilder sb = new StringBuilder(); List<ObjectError> errors = result.getAllErrors(); for (ObjectError error : errors) { logger.debug("Code: " + error.getCode()); logger.debug(StringUtils.join(error.getCodes(), "@#$^")); String code = (error.getCodes() != null && error.getCodes().length > 0) ? error.getCodes()[0] : null; logger.debug("The code in use:" + code); sb.append(messageSource.getMessage(code, null, error.getDefaultMessage(), null)).append("<br/>"); } return sb.toString(); } private List<MediaDTO> getSoundDtos(OccurrenceDTO occ) { String[] sounds = occ.getProcessed().getOccurrence().getSounds(); List<MediaDTO> soundDtos = new ArrayList<MediaDTO>(); if (sounds != null && sounds.length > 0) { for (String soundFile : sounds) { MediaDTO m = new MediaDTO(); Map<String, String> mimeToUrl = Config.mediaStore().getSoundFormats(soundFile); for (String mimeType : mimeToUrl.keySet()) { m.getAlternativeFormats().put(mimeType, mimeToUrl.get(mimeType)); } soundDtos.add(m); } } return soundDtos; } private void setupImageUrls(OccurrenceDTO dto) { String[] images = dto.getProcessed().getOccurrence().getImages(); if (images != null && images.length > 0) { List<MediaDTO> ml = new ArrayList<MediaDTO>(); for (String fileNameOrID : images) { MediaDTO m = new MediaDTO(); Map<String, String> urls = Config.mediaStore().getImageFormats(fileNameOrID); m.getAlternativeFormats().put("thumbnailUrl", urls.get("thumb")); m.getAlternativeFormats().put("smallImageUrl", urls.get("small")); m.getAlternativeFormats().put("largeImageUrl", urls.get("large")); m.getAlternativeFormats().put("imageUrl", urls.get("raw")); m.setFilePath(fileNameOrID); ml.add(m); } dto.setImages(ml); } } public void setSearchDAO(SearchDAO searchDAO) { this.searchDAO = searchDAO; } public void setSearchUtils(SearchUtils searchUtils) { this.searchUtils = searchUtils; } public void setSpeciesLookupService(SpeciesLookupService speciesLookupService) { this.speciesLookupService = speciesLookupService; } public void setContactUtils(ContactUtils contactUtils) { this.contactUtils = contactUtils; } public void setAssertionUtils(AssertionUtils assertionUtils) { this.assertionUtils = assertionUtils; } }