Java tutorial
/** * Copyright (C) 2014 Cohesive Integrations, LLC (info@cohesiveintegrations.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package net.di2e.ecdr.source.rest; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.net.URI; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import javax.activation.MimeType; import javax.activation.MimeTypeParseException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.cxf.jaxrs.client.WebClient; import org.apache.cxf.jaxrs.ext.multipart.ContentDisposition; import org.codice.ddf.security.common.jaxrs.RestSecurity; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.FrameworkUtil; import org.osgi.framework.InvalidSyntaxException; import org.osgi.framework.ServiceReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ddf.catalog.data.ContentType; import ddf.catalog.data.Metacard; import ddf.catalog.data.Result; import ddf.catalog.data.impl.ResultImpl; import ddf.catalog.filter.FilterAdapter; import ddf.catalog.operation.Query; import ddf.catalog.operation.QueryRequest; import ddf.catalog.operation.ResourceResponse; import ddf.catalog.operation.SourceResponse; import ddf.catalog.operation.impl.ResourceRequestByProductUri; import ddf.catalog.operation.impl.ResourceResponseImpl; import ddf.catalog.operation.impl.SourceResponseImpl; import ddf.catalog.resource.ResourceNotFoundException; import ddf.catalog.resource.ResourceNotSupportedException; import ddf.catalog.resource.impl.ResourceImpl; import ddf.catalog.service.ConfiguredService; import ddf.catalog.source.ConnectedSource; import ddf.catalog.source.FederatedSource; import ddf.catalog.source.SourceMonitor; import ddf.catalog.source.UnsupportedQueryException; import ddf.security.SecurityConstants; import ddf.security.Subject; import net.di2e.ecdr.api.cache.CacheManager; import net.di2e.ecdr.api.queryresponse.SearchResponseTransformer; import net.di2e.ecdr.commons.constants.SearchConstants; import net.di2e.ecdr.commons.filter.StrictFilterDelegate; import net.di2e.ecdr.search.transform.atom.response.AtomResponseTransformer; public class CDROpenSearchSource extends CDRSourceConfiguration implements FederatedSource, ConnectedSource, ConfiguredService { private static final Logger LOGGER = LoggerFactory.getLogger(CDROpenSearchSource.class); private static final String HEADER_ACCEPT_RANGES = "Accept-Ranges"; private static final String HEADER_CONTENT_DISPOSITION = "Content-Disposition"; private static final String HEADER_RANGE = "Range"; private static final String BYTES_TO_SKIP = "BytesToSkip"; private static final String BYTES_SKIPPED_RESPONSE = "BytesSkipped"; private static final String BYTES = "bytes"; private static final String BYTES_EQUAL = "bytes="; private SourceMonitor sourceMonitor = null; private FilterAdapter filterAdapter = null; private Date lastAvailableCheckDate = null; private boolean isCurrentlyAvailable = false; private String localId = null; private WebClient cdrRestClient = null; private WebClient cdrAvailabilityCheckClient = null; private String configurationPid = null; public CDROpenSearchSource(FilterAdapter adapter, CacheManager<Metacard> manager) { super(manager); this.filterAdapter = adapter; } @Override public SourceResponse query(QueryRequest queryRequest) throws UnsupportedQueryException { try { Query query = queryRequest.getQuery(); SourceResponse sourceResponse; // ECDR-72 Add in default radius Map<String, String> filterParameters = filterAdapter.adapt(query, new StrictFilterDelegate(false, getSupportedGeoOption(), getPropertyMap(), getDateTypeMap())); String id = filterParameters.get(SearchConstants.UID_PARAMETER); // check if this is an id-only query if (StringUtils.isBlank(id)) { // non-id query, perform normal search sourceResponse = doQuery(filterParameters, queryRequest); } else { // id-only query, check if remote source supports it if (supportsQueryById()) { sourceResponse = doQuery(filterParameters, queryRequest); } else { sourceResponse = lookupById(queryRequest, id); } } return sourceResponse; } catch (Exception e) { LOGGER.error(e.getMessage(), e); throw new UnsupportedQueryException( "Could not complete query to site [" + localId + "] due to: " + e.getMessage(), e); } } protected SourceResponse doQuery(Map<String, String> filterParameters, QueryRequest queryRequest) throws UnsupportedQueryException { SourceResponse sourceResponse; SearchResponseTransformer transformer = lookupSearchResponseTransformer(); setSecurityCredentials(cdrRestClient, queryRequest.getProperties()); filterParameters.putAll(getInitialFilterParameters(queryRequest)); setURLQueryString(filterParameters); setHttpHeaders(filterParameters, cdrRestClient); LOGGER.debug("Executing http GET query to source [{}] with url [{}]", localId, cdrRestClient.getCurrentURI().toString()); // TLSUtil.setTLSOptions( cdrRestClient ); Response response = cdrRestClient.get(); LOGGER.debug("Query to source [{}] returned http status code [{}] and media type [{}]", localId, response.getStatus(), response.getMediaType()); if (response.getStatus() == Status.OK.getStatusCode()) { // Be sure to pass in the getId() instead of the localId so Connected sources populate the Metacard with the // right Id sourceResponse = transformer.processSearchResponse((InputStream) response.getEntity(), queryRequest, getId()); if (!supportsQueryById()) { sourceResponse = cacheResults(sourceResponse); } } else { Object entity = response.getEntity(); if (entity != null) { try { LOGGER.warn("Error status code received [{}] when querying site [{}]:{}[{}]", response.getStatus(), localId, System.lineSeparator(), IOUtils.toString((InputStream) entity)); } catch (IOException e) { LOGGER.warn("Error status code received [{}] when querying site [{}]", response.getStatus(), localId); } } else { LOGGER.warn("Error status code received [{}] when querying site [{}]", response.getStatus(), localId); } throw new UnsupportedQueryException( "Query to remote source returned http status code " + response.getStatus()); } return sourceResponse; } @Override public boolean isAvailable() { LOGGER.debug( "isAvailable method called on CDR Rest Source named [{}], determining whether to check availability or pull from cache", localId); if (getPingMethod() != null && !PingMethod.NONE.equals(getPingMethod()) && cdrAvailabilityCheckClient != null) { if (!isCurrentlyAvailable || (lastAvailableCheckDate.getTime() < System.currentTimeMillis() - getAvailableCheckCacheTime())) { LOGGER.debug( "Checking availability on CDR Rest Source named [{}] in real time by calling endpoint [{}]", localId, cdrAvailabilityCheckClient.getBaseURI()); try { Response response = PingMethod.HEAD.equals(getPingMethod()) ? getPingClient().head() : getPingClient().get(); if (response.getStatus() == Status.OK.getStatusCode() || response.getStatus() == Status.ACCEPTED.getStatusCode()) { isCurrentlyAvailable = true; lastAvailableCheckDate = new Date(); } else { isCurrentlyAvailable = false; } } catch (RuntimeException e) { LOGGER.warn("CDR Rest Source named [" + localId + "] encountered an unexpected error while executing HTTP Head at URL [" + cdrAvailabilityCheckClient.getBaseURI() + "]:" + e.getMessage()); LOGGER.debug("Exception while trying to check avilability of site {}", localId, e); isCurrentlyAvailable = false; } } else { LOGGER.debug( "Pulling availability of CDR Rest Federated Source named [{}] from cache, isAvailable=[{}]", localId, isCurrentlyAvailable); } //if ( CollectionUtils.isNotEmpty( sourceMonitors ) ) { if (sourceMonitor != null) { // for( SourceMonitor sourceMonitor : sourceMonitors ){ if (isCurrentlyAvailable) { sourceMonitor.setAvailable(); } else { sourceMonitor.setUnavailable(); } // } } } else { LOGGER.debug( "HTTP Ping is set to false so not checking the sites availability, just setting to available"); isCurrentlyAvailable = true; //if ( CollectionUtils.isNotEmpty( sourceMonitors ) ) { if (sourceMonitor != null) { // for( SourceMonitor sourceMonitor : sourceMonitors ){ sourceMonitor.setAvailable(); // } } } return isCurrentlyAvailable; } @Override public void setId(String id) { LOGGER.debug("ConfigUpdate: Updating site name to [{}] by setId method", id); super.setId(id); localId = id; } @Override public void setShortname(String id) { LOGGER.debug("ConfigUpdate: Updating site name to [{}] by setShortname method", id); super.setId(id); localId = id; } @Override public boolean isAvailable(SourceMonitor callback) { sourceMonitor = callback; return isAvailable(); } @Override public Set<ContentType> getContentTypes() { return Collections.emptySet(); } @Override public Set<String> getOptions(Metacard paramMetacard) { return Collections.emptySet(); } @Override public Set<String> getSupportedSchemes() { return Collections.emptySet(); } @Override public ResourceResponse retrieveResource(URI uri, Map<String, Serializable> requestProperties) throws ResourceNotFoundException, ResourceNotSupportedException, IOException { LOGGER.debug("Retrieving Resource from remote CDR Source named [{}] using URI [{}]", localId, uri); // Check to see if the resource-uri value was passed through which is // the original metacard uri which // can be different from what was returned or used by the client Serializable resourceUriProperty = requestProperties.get(Metacard.RESOURCE_URI); if (resourceUriProperty != null && resourceUriProperty instanceof URI) { URI resourceUri = (URI) resourceUriProperty; if (!resourceUri.equals(uri)) { LOGGER.debug( "Overriding the passed in resourceUri [{}] with the value found in the request properties [{}]", uri, resourceUri); uri = resourceUri; } } else if (uri != null) { String scheme = uri.getScheme(); if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) { uri = getURIFromMetacard(uri); } } ResourceResponse resourceResponse = null; if (uri != null) { LOGGER.debug("Retrieving the remote resource using the uri [{}]", uri); WebClient retrieveWebClient = WebClient.create(uri); WebClient.getConfig(retrieveWebClient).getHttpConduit(); TLSUtil.setTLSOptions(retrieveWebClient, getDisableCNCheck()); resourceResponse = doRetrieval(retrieveWebClient, requestProperties); } if (resourceResponse == null) { LOGGER.warn("Could not retrieve resource from CDR Source named [{}] using uri [{}]", localId, uri); throw new ResourceNotFoundException( "Could not retrieve resource from source [" + localId + "] and uri [" + uri + "]"); } return resourceResponse; } protected ResourceResponse doRetrieval(WebClient retrieveWebClient, Map<String, Serializable> requestProperties) throws ResourceNotFoundException, IOException { ResourceResponse resourceResponse = null; setSecurityCredentials(retrieveWebClient, requestProperties); URI uri = retrieveWebClient.getCurrentURI(); try { Long bytesToSkip = null; // If a bytesToSkip property is present add range header if (requestProperties != null && requestProperties.containsKey(BYTES_TO_SKIP)) { bytesToSkip = (Long) requestProperties.get(BYTES_TO_SKIP); if (bytesToSkip != null) { LOGGER.debug( "Setting Range header on retrieve request from remote CDR Source [{}] with bytes to skip [{}]", localId, bytesToSkip); // This creates a Range header in the following manner if // 100 bytes were to be skipped. The end - means its open // ended // Range: bytes=100- retrieveWebClient.header(HEADER_RANGE, BYTES_EQUAL + bytesToSkip + "-"); } } Response clientResponse = retrieveWebClient.get(); MediaType mediaType = clientResponse.getMediaType(); MimeType mimeType = null; try { mimeType = (mediaType == null) ? new MimeType("application/octet-stream") : new MimeType(mediaType.toString()); LOGGER.debug( "Creating mime type from CDR Source named [{}] using uri [{}] with value [{}] defaulting to [{}]", localId, uri, mediaType); } catch (MimeTypeParseException e) { try { mimeType = new MimeType("application/octet-stream"); LOGGER.warn( "Creating mime type from CDR Source named [{}] using uri [{}] with value [{}] defaulting to [{}]", localId, uri, "application/octet-stream"); } catch (MimeTypeParseException e1) { LOGGER.error("Could not create MIMEType for resource being retrieved", e1); } } String dispositionString = clientResponse.getHeaderString(HEADER_CONTENT_DISPOSITION); String fileName = null; if (dispositionString != null) { ContentDisposition contentDisposition = new ContentDisposition(dispositionString); fileName = contentDisposition.getParameter("filename"); if (fileName == null) { fileName = contentDisposition.getParameter("\"filename\""); } if (fileName == null) { // ECDR-74 use MIMEType parser to get the file extension in fileName = getId() + "-" + System.currentTimeMillis(); } } else { // ECDR-74 use MIMEType parser to get the file extension in this // case fileName = getId() + "-" + System.currentTimeMillis(); } InputStream binaryStream = (InputStream) clientResponse.getEntity(); if (binaryStream != null) { Map<String, Serializable> responseProperties = new HashMap<String, Serializable>(); if (bytesToSkip != null) { // Since we sent a range header an accept-ranges header // should be returned if the // remote endpoint support it. If is not present, the // inputStream hasn't skipped ahead // by the given number of bytes, so we need to take care of // it here. String rangeHeader = clientResponse.getHeaderString(HEADER_ACCEPT_RANGES); if (rangeHeader == null || !rangeHeader.equals(BYTES)) { LOGGER.debug( "Skipping {} bytes in CDR Remote Source because endpoint didn't support Range Headers", bytesToSkip); try { // the Java inputStream.skip() method is not // guaranteed to skip all the bytes so we use a // utility method that is IOUtils.skipFully(binaryStream, bytesToSkip); } catch (EOFException e) { LOGGER.warn( "Skipping the requested number of bytes [{}] for URI [{}] resulted in an End of File, so re-retrieving the complete file without skipping bytes: {}", bytesToSkip, uri, e.getMessage()); try { binaryStream.close(); } catch (IOException e1) { LOGGER.debug("Error encountered while closing inputstream"); } return doRetrieval(retrieveWebClient, null); } } else if (rangeHeader != null && rangeHeader.equals(BYTES)) { LOGGER.debug( "CDR Remote source supports Range Headers, only retrieving part of file that has not been downloaded yet."); responseProperties.put(BYTES_SKIPPED_RESPONSE, Boolean.TRUE); } } resourceResponse = new ResourceResponseImpl(new ResourceRequestByProductUri(uri, requestProperties), responseProperties, new ResourceImpl(binaryStream, mimeType, fileName)); } } catch (RuntimeException e) { LOGGER.warn( "Expected exception encountered when trying to retrieve resource with URI [{}] from source [{}]", uri, localId); } return resourceResponse; } protected SearchResponseTransformer lookupSearchResponseTransformer() throws UnsupportedQueryException { SearchResponseTransformer transformer; if (StringUtils.isBlank(getResponseTransformerName())) { transformer = new AtomResponseTransformer(getAtomResponseTransformerConfig()); LOGGER.debug("Using the default Atom Response Transformer to transform response from site [{}]", localId); } else { transformer = getSearchResponseTransformer(getResponseTransformerName()); } if (transformer == null) { throw new UnsupportedQueryException("The query was not executed on the source " + localId + " because the response transformer was not a valid value [" + getResponseTransformerName() + "]. Please check the source configuration value for 'Response Transformer Override'"); } return transformer; } protected void setURLQueryString(Map<String, String> filterParameters) { cdrRestClient.resetQuery(); for (Entry<String, String> entry : filterParameters.entrySet()) { String parameterName = getParameterMap().get(entry.getKey()); if (StringUtils.isNotBlank(parameterName)) { cdrRestClient.replaceQueryParam(parameterName, entry.getValue()); } } Map<String, String> hardcodedQueryParams = getHardcodedQueryParameters(); for (Entry<String, String> entry : hardcodedQueryParams.entrySet()) { cdrRestClient.replaceQueryParam(entry.getKey(), entry.getValue()); } } protected Map<String, String> getInitialFilterParameters(QueryRequest request) { Map<String, String> filterParameters = new HashMap<String, String>(); Map<String, Serializable> queryRequestProps = request.getProperties(); if (LOGGER.isDebugEnabled()) { LOGGER.debug( "CDR REST Source received Query: " + ToStringBuilder.reflectionToString(request.getQuery())); } // include format parameter String format = (String) queryRequestProps.get(SearchConstants.FORMAT_PARAMETER); if (StringUtils.isNotBlank(format)) { filterParameters.put(SearchConstants.FORMAT_PARAMETER, format); } // Strict Mode Boolean strictMode = (Boolean) queryRequestProps.get(SearchConstants.STRICTMODE_PARAMETER); if (strictMode != null) { filterParameters.put(SearchConstants.STRICTMODE_PARAMETER, String.valueOf(strictMode)); } Query query = request.getQuery(); // Include timeout long timeout = query.getTimeoutMillis(); if (timeout > 1000) { filterParameters.put(SearchConstants.TIMEOUT_PARAMETER, String.valueOf(timeout)); } if (getParameterMap().containsKey(SearchConstants.COUNT_PARAMETER)) { int pageSize = query.getPageSize(); filterParameters.put(SearchConstants.COUNT_PARAMETER, getMaxResultsCount() > 0 && pageSize > getMaxResultsCount() ? String.valueOf(getMaxResultsCount()) : String.valueOf(pageSize)); } if (getParameterMap().containsKey(SearchConstants.STARTINDEX_PARAMETER)) { int startIndex = query.getStartIndex(); filterParameters.put(SearchConstants.STARTINDEX_PARAMETER, String.valueOf( getAtomResponseTransformerConfig().isZeroBasedStartIndex() ? startIndex - 1 : startIndex)); } String sortOrderString = getSortOrderString(query.getSortBy()); LOGGER.trace("Getting sort order for query [{}]", sortOrderString); if (sortOrderString != null) { filterParameters.put(SearchConstants.SORTKEYS_PARAMETER, sortOrderString); } for (Entry<String, Serializable> entry : queryRequestProps.entrySet()) { String key = entry.getKey(); if (getParameterMap().containsKey(key) || getHttpHeaderList().contains(key)) { String value = (String) entry.getValue(); if (StringUtils.isNotBlank(value)) { filterParameters.put(key, value); } } } LOGGER.trace("Filter Parameters being evaluated for inclusion in outgoing query {} which were parsed from", filterParameters); return filterParameters; } protected URI getURIFromMetacard(URI uri) { URI returnUri = null; Map<String, String> uriMap = new HashMap<String, String>(3); uriMap.put(Metacard.RESOURCE_URI, uri.toString()); setURLQueryString(uriMap); Response response = cdrRestClient.get(); AtomResponseTransformer transformer = new AtomResponseTransformer(getAtomResponseTransformerConfig()); SourceResponse sourceResponse = transformer.processSearchResponse((InputStream) response.getEntity(), null, getId()); List<Result> results = sourceResponse.getResults(); if (!results.isEmpty()) { returnUri = results.get(0).getMetacard().getResourceURI(); } return returnUri; } public SourceResponse cacheResults(SourceResponse sourceResponse) { for (Result result : sourceResponse.getResults()) { Metacard metacard = result.getMetacard(); getMetacardCache().put(metacard.getId(), metacard); } return sourceResponse; } private void setSecurityCredentials(WebClient client, Map<String, Serializable> requestProperties) { if (isSendSecurityCookie()) { if (requestProperties.containsKey(SecurityConstants.SECURITY_SUBJECT)) { Serializable property = requestProperties.get(SecurityConstants.SECURITY_SUBJECT); if (property instanceof Subject) { Subject subject = (Subject) property; RestSecurity.setUnsecuredSubjectOnClient(subject, client); } } } } protected SearchResponseTransformer getSearchResponseTransformer(String id) { SearchResponseTransformer transformer = null; Bundle bundle = FrameworkUtil.getBundle(this.getClass()); if (bundle != null) { BundleContext context = bundle.getBundleContext(); Collection<ServiceReference<SearchResponseTransformer>> transformers; try { transformers = context.getServiceReferences(SearchResponseTransformer.class, "(id=" + id + ")"); int size = transformers.size(); if (size > 0) { transformer = context.getService(transformers.iterator().next()); if (size > 1) { LOGGER.debug( "Multiple [{}] InputTransformers were returned when looking up InputTransformer with id [{}], using the first one {}", size, id, transformer.getClass().getName()); } } } catch (InvalidSyntaxException e) { LOGGER.warn("Could not lookup input transformer with id [{}]", id, e.getMessage()); } } return transformer; } protected SourceResponse lookupById(QueryRequest queryRequest, String id) throws UnsupportedQueryException { SourceResponse sourceResponse = null; LOGGER.debug("Checking cache for Result with id [{}].", id); Metacard metacard = getMetacardCache().get(id); if (metacard != null) { metacard.setSourceId(getId()); LOGGER.debug("Cache hit found for id [{}], returning response", id); sourceResponse = new SourceResponseImpl(queryRequest, Arrays.asList((Result) new ResultImpl(metacard)), 1L); } else { LOGGER.debug("Could not find result id [{}] in cache", id); throw new UnsupportedQueryException( "Queries for parameter uid are not supported by source [" + localId + "]"); } return sourceResponse; } protected void setHttpHeaders(Map<String, String> filterParameters, WebClient client) { Map<String, String> hardcodedHeaders = getHardcodedHttpHeaders(); if (MapUtils.isNotEmpty(hardcodedHeaders)) { for (Entry<String, String> entry : hardcodedHeaders.entrySet()) { String key = entry.getKey(); String value = entry.getValue(); client.header(key, value); LOGGER.trace("Adding the following HTTP Header to outgoing request [{}]=[{}]", key, value); } } List<String> headers = getHttpHeaderList(); if (CollectionUtils.isNotEmpty(headers)) { for (String header : headers) { if (filterParameters.containsKey(header)) { String value = filterParameters.get(header); client.header(header, value); LOGGER.trace("Adding the following HTTP Header to outgoing request [{}]=[{}]", header, value); } } } } protected boolean supportsQueryById() { return getParameterMap().containsKey(SearchConstants.UID_PARAMETER); } @Override protected WebClient getRestClient() { return cdrRestClient; } @Override protected void setRestClient(WebClient webClient) { cdrRestClient = webClient; } @Override protected WebClient getPingClient() { return cdrAvailabilityCheckClient; } @Override protected void setPingClient(WebClient webClient) { cdrAvailabilityCheckClient = webClient; } @Override public String getConfigurationPid() { return configurationPid; } @Override public void setConfigurationPid(String pid) { configurationPid = pid; } }