Java tutorial
/** ** Copyright 2016 General Electric Company ** ** ** Licensed under the Apache License, Version 2.0 (the "License"); ** you may not use this file except in compliance with the License. ** You may obtain a copy of the License at ** ** http://www.apache.org/licenses/LICENSE-2.0 ** ** Unless required by applicable law or agreed to in writing, software ** distributed under the License is distributed on an "AS IS" BASIS, ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ** See the License for the specific language governing permissions and ** limitations under the License. */ package com.ge.research.semtk.sparqlX; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLStreamHandler; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.codec.net.URLCodec; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.AuthCache; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.ClientContext; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.impl.auth.DigestScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.util.EntityUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.JSONValue; import org.json.simple.parser.JSONParser; import com.ge.research.semtk.load.utility.SparqlGraphJson; import com.ge.research.semtk.resultSet.GeneralResultSet; import com.ge.research.semtk.resultSet.NodeGroupResultSet; import com.ge.research.semtk.resultSet.SimpleResultSet; import com.ge.research.semtk.resultSet.Table; import com.ge.research.semtk.resultSet.TableResultSet; import com.ge.research.semtk.sparqlX.FusekiSparqlEndpointInterface; import com.ge.research.semtk.sparqlX.VirtuosoSparqlEndpointInterface; /** * Interface to SPARQL endpoint. * This is an abstract class - create a subclass per implementation (Virtuoso, etc) */ @SuppressWarnings("deprecation") public abstract class SparqlEndpointInterface { // NOTE: more than one thread cannot safely share a SparqlEndpointInterface. // this is because of the state maintained for the results vars and connection // details. doing so may lead to unexpected results // results types to request private static final String CONTENTTYPE_SPARQL_QUERY_RESULT_JSON = "application/sparql-results+json"; private static final String CONTENTTYPE_JSON_LD = "application/x-json+ld"; private static final int MAX_QUERY_TRIES = 3; private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd"); private static URLStreamHandler handler = null; // set only by unit tests private JSONObject response = null; private JSONArray resVars = null; private JSONArray resBindings = null; protected String userName = null; protected String password = null; protected String server = null; protected String port = null; protected String dataset = ""; /** * Constructor * @param serverAndPort e.g. "http://vesuvius37.crd.ge.com:2420" * @param dataset e.g. "http://research.ge.com/energy/dataset" * @throws Exception */ public SparqlEndpointInterface(String serverAndPort, String dataset) throws Exception { this(serverAndPort, dataset, null, null); } /** * Constructor used for authenticated connections... like insert/update/delete/clear */ public SparqlEndpointInterface(String serverAndPort, String dataset, String user, String pass) throws Exception { this.dataset = dataset; this.userName = user; this.password = pass; this.setServerAndPort(serverAndPort); } public String getServerAndPort() { return this.server + ":" + this.port; } public void setServerAndPort(String serverAndPort) throws Exception { String[] serverAndPortSplit = serverAndPort.split(":"); // protocol:server:port if (serverAndPortSplit.length < 2) { throw new Exception( "Error: must provide connection in format protocol:server:port (e.g. http://vesuvius37.crd.ge.com:2420)"); } this.server = serverAndPortSplit[0] + ":" + serverAndPortSplit[1]; // e.g. http://vesuvius37.crd.ge.com if (serverAndPortSplit.length < 3) { throw new Exception("Error: no port provided for " + this.server); } String[] portandendpoint = serverAndPortSplit[2].split("/"); this.port = portandendpoint[0]; } public String getServer() { return server; } public int getPort() { return (int) Integer.parseInt(port); } public String getDataset() { return this.dataset; } public String getUserName() { return userName; } public String getPassword() { return password; } public void setDataset(String dataset) { this.dataset = dataset; } public abstract String getServerType(); /** * Build a GET URL (implementation-specific) */ public abstract String getPostURL(); /** * Build an upload URL (implementation-specific) */ public abstract String getUploadURL() throws Exception; /** * Build a POST URL (implementation-specific) */ public abstract String getGetURL(); /** * Handle an empty response (implementation-specific) */ public abstract void handleEmptyResponse() throws Exception; /** * Get a default result type */ private static SparqlResultTypes getDefaultResultType() { return SparqlResultTypes.TABLE; // if no result type, assume it's a SELECT } public static void setUrlStreamHandler(URLStreamHandler handler) { SparqlEndpointInterface.handler = handler; } /** * Set the username and password */ public void setUserAndPassword(String user, String pass) { this.userName = user; this.password = pass; } /** * Static method to get an instance of this abstract class */ public static SparqlEndpointInterface getInstance(String serverTypeString, String server, String dataset) throws Exception { return getInstance(serverTypeString, server, dataset, null, null); } /** * Static method to get an instance of this abstract class */ public static SparqlEndpointInterface getInstance(String serverTypeString, String server, String dataset, String user, String password) throws Exception { if (serverTypeString.equalsIgnoreCase("virtuoso")) { if (user != null && password != null) { return new VirtuosoSparqlEndpointInterface(server, dataset, user, password); } else { return new VirtuosoSparqlEndpointInterface(server, dataset); } } else if (serverTypeString.equalsIgnoreCase("fuseki")) { if (user != null && password != null) { return new FusekiSparqlEndpointInterface(server, dataset, user, password); } else { return new FusekiSparqlEndpointInterface(server, dataset); } } else { throw new Exception("Invalid SPARQL server type : " + serverTypeString); } } // I renamed this function because it was not clear which Endpoint it was getting // And labeled it deprecated because it is only two lines of code and the first one is usually shared with calls to retrieve other things from the json. // And re-implemented it with the preferred code. // - Paul 5/26/2016 public static SparqlEndpointInterface getDataInterfaceFromJsonDEPRECATE(JSONObject json) throws Exception { SparqlGraphJson sgJson = new SparqlGraphJson(json); SparqlEndpointInterface sei = sgJson.getSparqlConn().getDataInterface(0); return sei; } /** * Static method to create an endpoint and execute a query */ public static SparqlEndpointInterface executeQuery(String server, String dataset, String serverTypeString, String query, SparqlResultTypes resultType) throws Exception { return executeQuery(server, dataset, serverTypeString, query, null, null, resultType); } /** * Static method to create an endpoint and execute a query */ public static SparqlEndpointInterface executeQuery(String server, String dataset, String serverTypeString, String query, String user, String pass, SparqlResultTypes resultType) throws Exception { try { SparqlEndpointInterface endpoint; if (user != null && pass != null) { endpoint = SparqlEndpointInterface.getInstance(serverTypeString, server, dataset, user, pass); } else { endpoint = SparqlEndpointInterface.getInstance(serverTypeString, server, dataset); } endpoint.executeQuery(query, resultType); return endpoint; } catch (Exception e) { e.printStackTrace(); throw new Exception("Error executing semantic query:\n\t" + e.getMessage()); } } /** * Execute query and assemble a GeneralResultSet object. * * Returns: * TableResultSet for a select query (success or failure) * NodeGroupResultSet for a construct query (success or failure) * SimpleResultSet for an auth query (success or failure) */ public GeneralResultSet executeQueryAndBuildResultSet(String query, SparqlResultTypes resultType) { // construct an empty result object to store the results GeneralResultSet resultSet = null; try { if (resultType == SparqlResultTypes.GRAPH_JSONLD) { resultSet = new NodeGroupResultSet(); // construct queries produce graphs } else if (resultType == SparqlResultTypes.CONFIRM) { resultSet = new SimpleResultSet(); // auth queries produce messages } else if (resultType == SparqlResultTypes.TABLE) { resultSet = new TableResultSet(); // non-construct queries produce tables } } catch (Exception e) { // should never get here, but if it does it will return a SimpleResultSet with an error e.printStackTrace(); resultSet = new SimpleResultSet(false, e.getMessage()); } try { // execute the query JSONObject result = executeQuery(query, resultType); resultSet.setSuccess(true); resultSet.addResultsJSON(result); } catch (Exception e) { // error executing query - return the appropriate ResultSet subtype with failure and message e.printStackTrace(); resultSet.setSuccess(false); resultSet.addRationaleMessage(e.getMessage()); } return resultSet; } /** * Execute a query. Uses http POST. * Depending on the type of query (select, insert, construct, etc), decides whether to use auth or non-auth. * * Sample result from a SELECT query: * * Sample result from a CONSTRUCT query: * * Sample results from an AUTH query: * {"@message":"Insert into <http://research.ge.com/dataset>, 1 (or less) triples -- done"} * * @param query * @return a JSONObject wrapping the results */ public JSONObject executeQuery(String query, SparqlResultTypes resultType) throws Exception { int tryCount = 0; // Keep trying the query until it succeeds or reaches a // maximum number of tries, which is checked for in the // exception catch while (true) { tryCount++; try { if (this.userName != null && this.password != null) { return executeQueryAuthPost(query, resultType); } else { if (resultType == SparqlResultTypes.CONFIRM) { throw new Exception("Username and password are required to execute a query with resultType " + resultType.toString()); } return executeQueryPost(query, resultType); } } catch (Exception e) { if (tryCount >= MAX_QUERY_TRIES) { throw e; } else { // else unnecessary, but makes code easier to read System.out.println("SPARQL query failed. Sleeping 2 seconds and trying again..."); TimeUnit.SECONDS.sleep(2); // sleep 2 seconds and try again } } } } public JSONObject executeTestQuery() throws Exception { final String sparql = "select ?Concept where {[] a ?Concept} LIMIT 1"; try { return executeQuery(sparql, SparqlResultTypes.TABLE); } catch (Exception e) { throw new Exception("Failure executing test query.", e); } } /** * Execute query using POST * @return a JSONObject wrapping the results. in the event the results were tabular, they can be obtained in the JsonArray "@Table". if the results were a graph, use "@Graph" for json-ld * @throws Exception * * Sample table output: * * Sample graph output: */ public JSONObject executeQueryPost(String query, SparqlResultTypes resultType) throws Exception { if (resultType == null) { resultType = getDefaultResultType(); } // execute a query and return a JsonArray of results. // May throw Exceptions for various simple errors String resultsFormat = this.getContentType(resultType); DefaultHttpClient httpclient = new DefaultHttpClient(); String[] serverNoProtocol = this.server.split("://"); HttpHost targetHost = new HttpHost(serverNoProtocol[1], Integer.valueOf(this.port), serverNoProtocol[0]); HttpPost httppost = new HttpPost(getPostURL()); httppost.addHeader("Accept", resultsFormat); httppost.addHeader("X-Sparql-default-graph", this.dataset); // add params List<NameValuePair> params = new ArrayList<>(3); params.add(new BasicNameValuePair("query", query)); params.add(new BasicNameValuePair("format", resultsFormat)); params.add(new BasicNameValuePair("default-graph-uri", this.dataset)); httppost.setEntity(new UrlEncodedFormEntity(params, "UTF-8")); HttpResponse httpresponse = httpclient.execute(targetHost, httppost); HttpEntity entity = httpresponse.getEntity(); String responseTxt = EntityUtils.toString(entity, "UTF-8"); try { this.response = (JSONObject) new JSONParser().parse(responseTxt); } catch (Exception e) { entity.getContent().close(); throw new Exception("Cannot parse query result into JSON: " + responseTxt); } JSONObject interimObj = new JSONObject(this.response); if (this.response == null) { System.err.println("the response could not be transformed into json"); entity.getContent().close(); return null; } else { JSONObject procResp = getResultsFromResponse(interimObj, resultType); entity.getContent().close(); return procResp; } } /** * Execute an auth query using POST * @return a JSONObject wrapping the results. in the event the results were tabular, they can be obtained in the JsonArray "@Table". if the results were a graph, use "@Graph" for json-ld * @throws Exception */ private JSONObject executeQueryAuthPost(String query, SparqlResultTypes resultType) throws Exception { if (resultType == null) { resultType = getDefaultResultType(); } DefaultHttpClient httpclient = new DefaultHttpClient(); httpclient.getCredentialsProvider().setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.userName, this.password)); String[] serverNoProtocol = this.server.split("://"); //System.err.println("the new server name is: " + serverNoProtocol[1]); HttpHost targetHost = new HttpHost(serverNoProtocol[1], Integer.valueOf(this.port), "http"); DigestScheme digestAuth = new DigestScheme(); AuthCache authCache = new BasicAuthCache(); digestAuth.overrideParamter("realm", "SPARQL"); // Suppose we already know the expected nonce value digestAuth.overrideParamter("nonce", "whatever"); authCache.put(targetHost, digestAuth); BasicHttpContext localcontext = new BasicHttpContext(); localcontext.setAttribute(ClientContext.AUTH_CACHE, authCache); // add new stuff HttpPost httppost = new HttpPost(getPostURL()); String resultsFormat = this.getContentType(resultType); httppost.addHeader("Accept", resultsFormat); httppost.addHeader("X-Sparql-default-graph", this.dataset); // add params List<NameValuePair> params = new ArrayList<NameValuePair>(3); params.add(new BasicNameValuePair("query", query)); params.add(new BasicNameValuePair("format", resultsFormat)); params.add(new BasicNameValuePair("default-graph-uri", this.dataset)); httppost.setEntity(new UrlEncodedFormEntity(params, "UTF-8")); // finish new stuff HttpResponse response_http = httpclient.execute(targetHost, httppost, localcontext); HttpEntity entity = response_http.getEntity(); String responseTxt = EntityUtils.toString(entity, "UTF-8"); // some diagnostic output if (responseTxt == null) { System.err.println("the response text was null!"); } if (responseTxt.trim().isEmpty()) { handleEmptyResponse(); // implementation-specific behavior } JSONObject resp; try { resp = (JSONObject) JSONValue.parse(responseTxt); } catch (Exception e) { entity.getContent().close(); throw new Exception("Cannot parse query result into JSON: " + responseTxt); } if (resp == null) { System.err.println("the response could not be transformed into json"); if (responseTxt.contains("Error")) { entity.getContent().close(); throw new Exception(responseTxt); } entity.getContent().close(); return null; } else { JSONObject procResp = getResultsFromResponse(resp, resultType); entity.getContent().close(); return procResp; } } /** * Execute an auth query using POST * @return a JSONObject wrapping the results. in the event the results were tabular, they can be obtained in the JsonArray "@Table". if the results were a graph, use "@Graph" for json-ld * @throws Exception */ public JSONObject executeAuthUploadOwl(byte[] owl) throws Exception { DefaultHttpClient httpclient = new DefaultHttpClient(); httpclient.getCredentialsProvider().setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.userName, this.password)); String[] serverNoProtocol = this.server.split("://"); //System.err.println("the new server name is: " + serverNoProtocol[1]); HttpHost targetHost = new HttpHost(serverNoProtocol[1], Integer.valueOf(this.port), "http"); DigestScheme digestAuth = new DigestScheme(); AuthCache authCache = new BasicAuthCache(); digestAuth.overrideParamter("realm", "SPARQL"); // Suppose we already know the expected nonce value digestAuth.overrideParamter("nonce", "whatever"); authCache.put(targetHost, digestAuth); BasicHttpContext localcontext = new BasicHttpContext(); localcontext.setAttribute(ClientContext.AUTH_CACHE, authCache); // add new stuff HttpPost httppost = new HttpPost(getUploadURL()); String resultsFormat = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; httppost.addHeader("Accept", resultsFormat); httppost.addHeader("X-Sparql-default-graph", this.dataset); MultipartEntityBuilder builder = MultipartEntityBuilder.create(); builder.addTextBody("graph-uri", this.dataset); builder.addBinaryBody("res-file", owl); HttpEntity entity = builder.build(); httppost.setEntity(entity); /* THIS IS THE MULTIPART FORMAT WE NEED TO SEND. Content-Type: multipart/form-data; boundary=---------------------------32932166721282 Content-Length: 234 -----------------------------32932166721282 Content-Disposition: form-data; name="graph-uri" http://www.kdl.ge.com/changeme -----------------------------32932166721282 Content-Disposition: form-data; name="res-file"; filename="employee.owl" Content-Type: application/octet-stream <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:owl="http://www.w3.org/2002/07/owl#" xmlns="http://kdl.ge.com/pd/employee#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" . . . </rdf:RDF> -----------------------------32932166721282-- */ executeTestQuery(); HttpResponse response_http = httpclient.execute(targetHost, httppost, localcontext); HttpEntity resp_entity = response_http.getEntity(); // get response with HTML tags removed String responseTxt = EntityUtils.toString(resp_entity, "UTF-8").replaceAll("\\<.*?>", " "); SimpleResultSet ret = new SimpleResultSet(); if (responseTxt.trim().isEmpty()) { // success or bad login :-( ret.setSuccess(true); } else { ret.setSuccess(false); ret.addRationaleMessage(responseTxt); } resp_entity.getContent().close(); return ret.toJson(); } /** * Execute query using GET (use should be rare - in cases where POST is not supported) * @return a JSONObject wrapping the results. in the event the results were tabular, they can be obtained in the JsonArray "@Table". if the results were a graph, use "@Graph" for json-ld * @throws Exception */ public JSONObject executeQueryGet(String query, SparqlResultTypes resultsType) throws Exception { if (resultsType == null) { resultsType = getDefaultResultType(); } // encode the query and build a real URL URLCodec encoder = new URLCodec(); String cleanURL = getGetURL() + encoder.encode(query); URL url = new URL(null, cleanURL, handler); System.out.println("URL: " + url); // create an http GET connection and make the request HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // set the type of output desired. this might not work.... String resultsFormat = this.getContentType(resultsType); conn.setRequestProperty("Accept", resultsFormat); conn.setRequestMethod("GET"); // read the results BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); String results = ""; // get everythign from reader String line; while ((line = rd.readLine()) != null) { results += line; } try { this.response = (JSONObject) new JSONParser().parse(results); } catch (Exception e) { throw new Exception("Cannot parse query result into JSON: " + results); } JSONObject interimObj = new JSONObject(this.response); rd.close(); // close the reader if (this.response == null) { System.err.println("the response could not be transformed into json"); return null; } else { JSONObject procResp = getResultsFromResponse(interimObj, resultsType); return procResp; } } /** * Execute an auth query using GET (use should be rare - in cases where POST is not supported) * @return a JSONObject wrapping the results. in the event the results were tabular, they can be obtained in the JsonArray "@Table". if the results were a graph, use "@Graph" for json-ld * @throws Exception */ @SuppressWarnings("unused") private JSONObject executeQueryAuthGet(String queryAndUrl, SparqlResultTypes resultType) throws Exception { if (resultType == null) { resultType = getDefaultResultType(); } DefaultHttpClient httpclient = new DefaultHttpClient(); //ResponseHandler<String> responseHandler = new BasicResponseHandler(); System.err.println("the server name was " + this.server); System.err.println("the port id was " + this.port); System.err.println("the user name was " + "SPARQL/" + this.userName); System.err.println("the password was " + this.password); System.err.println(queryAndUrl); httpclient.getCredentialsProvider().setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.userName, this.password)); String[] serverNoProtocol = this.server.split("://"); System.err.println("the new server name is: " + serverNoProtocol[1]); HttpHost targetHost = new HttpHost(serverNoProtocol[1], Integer.valueOf(this.port), "http"); DigestScheme digestAuth = new DigestScheme(); AuthCache authCache = new BasicAuthCache(); digestAuth.overrideParamter("realm", "SPARQL"); // Suppose we already know the expected nonce value digestAuth.overrideParamter("nonce", "whatever"); authCache.put(targetHost, digestAuth); BasicHttpContext localcontext = new BasicHttpContext(); localcontext.setAttribute(ClientContext.AUTH_CACHE, authCache); HttpGet httpget = new HttpGet(queryAndUrl); String resultsFormat = this.getContentType(resultType); httpget.addHeader("Accept", resultsFormat); System.out.println("executing request" + httpget.getRequestLine()); // String responseTxt = httpclient.execute(httpget, responseHandler); HttpResponse response_http = httpclient.execute(targetHost, httpget, localcontext); HttpEntity entity = response_http.getEntity(); String responseTxt = EntityUtils.toString(entity, "UTF-8"); // some diagnostic output if (responseTxt == null) { System.err.println("the response text was null!"); } if (responseTxt.trim().isEmpty()) { handleEmptyResponse(); // implementation-specific behavior } if (responseTxt.length() < 100) { System.err.println("SparqlEndpointInterface received: " + responseTxt); } else { System.err.println("SparqlEndpointInterface received: " + responseTxt.substring(0, 99) + "... (" + responseTxt.length() + " chars)"); } JSONObject resp; try { resp = (JSONObject) new JSONParser().parse(responseTxt); } catch (Exception e) { throw new Exception("Cannot parse query result into JSON: " + responseTxt); } if (resp == null) { System.err.println("the response could not be transformed into json"); if (responseTxt.contains("Error")) { throw new Exception(responseTxt); } return null; } else { JSONObject procResp = getResultsFromResponse(resp, resultType); return procResp; } } /** * Execute a query, retrieving a HashMap of data for a given set of headers. */ public HashMap<String, String[]> executeQuery(String query, String[] resultHeaders) throws Exception { return executeQuery(query, resultHeaders, false); } /** * Execute a query, retrieving a HashMap of data for a given set of headers. * * @param query the query * @param resultHeaders the headers of the result columns desired. If null, use all column headers. * @param expectOneResult if true, expect a single row result per column - if zero or multiple results are returned, then error * @return a hashmap: keys are resultHeaders, values are String arrays with contents * * TODO work on this - make it more generic */ public HashMap<String, String[]> executeQuery(String query, String[] resultHeaders, boolean expectOneResult) throws Exception { executeQuery(query, SparqlResultTypes.TABLE); HashMap<String, String[]> results = new HashMap<String, String[]>(); String colArr[] = null; if (resultHeaders != null) { colArr = resultHeaders; } else { List<String> colList = getResultsColumnName(); colArr = colList.toArray(new String[colList.size()]); } for (String resultHeader : colArr) { String[] column = getStringResultsColumn(resultHeader); if (expectOneResult) { // expect each column to have exactly one entry if (column.length > 1) { throw new Exception( "Expected 1 result for " + resultHeader + ", but retrieved multiple results"); } else if (column.length < 1) { throw new Exception("Expected 1 result for " + resultHeader + ", but retrieved zero results"); } } results.put(resultHeader, column); } return results; } public JSONObject getResponse() { return this.response; } /** * Get query results as ints */ public Integer[] getIntResultsColumn(String colName) throws Exception { Integer ret[] = null; String s = null; this.checkResultsCol(colName); // declare the array and populate it ret = new Integer[this.resBindings.size()]; for (int i = 0; i < this.resBindings.size(); i++) { if (((JSONObject) this.resBindings.get(i)).containsKey(colName) && ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).containsKey("value")) { s = (String) ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).get("value"); ret[i] = Integer.parseInt(s); } else { ret[i] = null; } } return ret; } /** * Get query results as Strings */ public String[] getStringResultsColumn(String colName) throws Exception { String ret[] = null; this.checkResultsCol(colName); // declare the array and populate it ret = new String[this.resBindings.size()]; for (int i = 0; i < this.resBindings.size(); i++) { if (((JSONObject) this.resBindings.get(i)).containsKey(colName) && ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).containsKey("value")) { ret[i] = (String) ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).get("value"); } else { ret[i] = null; } } return ret; } /** * Get query results as doubles */ public Double[] getDoubleResultsColumn(String colName) throws Exception { Double ret[] = null; String s = null; this.checkResultsCol(colName); // declare the array and populate it ret = new Double[this.resBindings.size()]; for (int i = 0; i < this.resBindings.size(); i++) { if (((JSONObject) this.resBindings.get(i)).containsKey(colName) && ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).containsKey("value")) { s = (String) ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).get("value"); ret[i] = Double.parseDouble(s); } else { ret[i] = null; } } return ret; } /** * Get query results as Dates */ public Date[] getDateResultsColumn(String colName) throws Exception { Date[] ret = null; String s = null; this.checkResultsCol(colName); // declare the array and populate it ret = new Date[this.resBindings.size()]; for (int i = 0; i < this.resBindings.size(); i++) { if (((JSONObject) this.resBindings.get(i)).containsKey(colName) && ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).containsKey("value")) { s = (String) ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(colName)).get("value"); ret[i] = SparqlEndpointInterface.DATE_FORMAT.parse(s); } else { ret[i] = null; } } return ret; } /** * Check if the query results contain a given column name (throw exception if not) */ public void checkResultsCol(String colName) throws Exception { int col = -1; // find column for (int i = 0; i < this.resVars.size(); i++) { if (((String) this.resVars.get(i)).equals(colName)) { col = i; break; } } if (col == -1) throw new Exception(String.format( "SparqlEndpointInterface: Asked for column %s which was not returned by the Sparql query.", colName)); } /** * Return all result column names. */ public ArrayList<String> getResultsColumnName() { ArrayList<String> retval = new ArrayList<String>(); for (int i = 0; i < this.resVars.size(); i++) { retval.add((String) (this.resVars.get(i))); } return retval; } // TODO this is currently unused - do we want to keep it? public String[] getColumnNamesNotInInputList(String[] inputList) { ArrayList<String> retval = new ArrayList<String>(); for (int i = 0; i < this.resVars.size(); i++) { retval.add((String) (this.resVars.get(i))); } // filter the ones we do not want. for (String i : inputList) { retval.remove(i); } String[] retval_array = new String[retval.size()]; retval.toArray(retval_array); return retval_array; } // TODO this is currently unused - do we want to keep it? It is similar to Table.getSubset() public ArrayList<String[]> getValuesForColsBasedOnCol1Key(String compareColName, String keyValue, String[] gatherCols) throws IOException { // uses the compareColName and searches for the keyValue. // once found, it returns the values of the other columns specified by name in gatherCols ArrayList<String[]> retval = new ArrayList<String[]>(); try { // check all exist this.checkResultsCol(compareColName); for (String i : gatherCols) { this.checkResultsCol(i); } // do actual work for (int i = 0; i < this.resBindings.size(); i++) { // find a row of interest if (((String) ((JSONObject) ((JSONObject) this.resBindings.get(i)).get(compareColName)) .get("value")).equals(keyValue)) { // System.err.println(keyValue + " found!"); String[] line = new String[gatherCols.length + 1]; // we include the key in there as well. line[0] = keyValue; for (int j = 0; j < gatherCols.length; j += 1) { // add the columns line[j + 1] = ((String) ((JSONObject) ((JSONObject) this.resBindings.get(i)) .get(gatherCols[j])).get("value")); } retval.add(line); // make sure this is in the return set. } } // System.err.println("_____________________________"); return retval; } catch (Exception e) { throw new IOException(e.getMessage()); } } /** * Get a results content type to be set in the HTTP header. */ private String getContentType(SparqlResultTypes resultType) throws Exception { if (resultType == SparqlResultTypes.TABLE || resultType == SparqlResultTypes.CONFIRM) { return CONTENTTYPE_SPARQL_QUERY_RESULT_JSON; } if (resultType == SparqlResultTypes.GRAPH_JSONLD) { return CONTENTTYPE_JSON_LD; } // fail and throw an exception if the value was not valid. throw new Exception("Cannot get content type for query type " + resultType); } // handle results based on expected type @SuppressWarnings("unchecked") private JSONObject getResultsFromResponse(JSONObject resp, SparqlResultTypes resultType) throws Exception { JSONObject retval = null; this.response = resp; // check for a result type that creates a table. // null is default assumption that one meant a tabular result. if (resultType == SparqlResultTypes.TABLE || resultType == SparqlResultTypes.CONFIRM) { JSONObject head = (JSONObject) response.get("head"); if (head == null) { throw new Exception("Sparql server did not return a 'head' object"); } this.resVars = (JSONArray) head.get("vars"); if (this.resVars == null) { throw new Exception("Sparql server response 'head' did not include a 'vars' array of column names"); } JSONObject results = (JSONObject) response.get("results"); if (results == null) { throw new Exception("Sparql server did not return a 'results' object"); } this.resBindings = (JSONArray) results.get("bindings"); if (this.resBindings == null) { throw new Exception( "Sparql server response 'results' did not include a 'bindings' array of result rows"); } retval = new JSONObject(); if (this.resBindings == null) { this.resBindings = new JSONArray(); } if (resultType == SparqlResultTypes.TABLE) { retval.put(TableResultSet.TABLE_JSONKEY, getTableJSONFromResponse(this.resVars, this.resBindings)); // @table } else { retval.put(SimpleResultSet.MESSAGE_JSONKEY, getAuthMessageFromResponse(this.resBindings)); // @message } } else if (resultType == SparqlResultTypes.GRAPH_JSONLD) { retval = new JSONObject(resp); } else { throw new Exception( "an unknown results type was passed to \"getResultsBasedOnExpectedType\". don't know how to handle type: " + resultType); } return retval; } /** * Parse an auth query confirmation. The message will the first entry in the rows array. * @param rowsJsonArray * @return */ private static String getAuthMessageFromResponse(JSONArray rowsJsonArray) { String message = ""; for (int i = 0; i < rowsJsonArray.size(); i++) { JSONObject row = (JSONObject) rowsJsonArray.get(i); // e.g. {"callret-0":{"type":"literal","value":"Insert into <http:\/\/research.ge.com\/dataset>, 1 (or less) triples -- done"}} JSONObject value; String key, valueValue; @SuppressWarnings("unchecked") Iterator<String> iter = row.keySet().iterator(); while (iter.hasNext()) { key = (String) iter.next(); value = (JSONObject) row.get(key); valueValue = (String) value.get("value"); message += valueValue; } } return message; } /** * Convert response table json to SemTK Table json * * Sample input SPARQL table json: *{"Test":{"type":"literal","value":"http:\/\/research.ge.com\/dataset#Test_1"},"number":{"datatype":"http:\/\/www.w3.org\/2001\/XMLSchema#integer","type":"typed-literal","value":"1272"}}, *{"Test":{"type":"literal","value":"http:\/\/research.ge.com\/dataset#Test_2"},"number":{"datatype":"http:\/\/www.w3.org\/2001\/XMLSchema#integer","type":"typed-literal","value":"1274"}}, *{"Test":{"type":"literal","value":"http:\/\/research.ge.com\/dataset#Test_3"},"number":{"datatype":"http:\/\/www.w3.org\/2001\/XMLSchema#integer","type":"typed-literal","value":"1276"}}, * * Sample output SemTK table json: * { * "col_names":["Test","testnum"], * "rows":[["http://research.ge.com/energy/dataset#Test_1272","1272"],["http://research.ge.com/dataset#Test_1274","1274"],["http://research.ge.com/dataset#Test_1276","1276"]], * "col_type":["uri","http:\/\/www.w3.org\/2001\/XMLSchema#integer"], // TODO is this correct? * "col_count":2, * "row_count":3 * } * * @param colNamesJsonArray the JSONArray containing the column names * @param rowsJsonArray the JSONArray containing the data rows * @return * @throws Exception */ private static JSONObject getTableJSONFromResponse(JSONArray colNamesJsonArray, JSONArray rowsJsonArray) throws Exception { String key, valueValue, valueType, valueDataType; JSONObject jsonCell; ArrayList<String> colsForNewTable = new ArrayList<String>(); ArrayList<String> colTypesForNewTable = new ArrayList<String>(); ArrayList<String> rowForNewTable; ArrayList<ArrayList<String>> rowsForNewTable = new ArrayList<ArrayList<String>>(); HashMap<String, Integer> colNumHash = new HashMap<String, Integer>(); HashMap<String, String> colTypeHash = new HashMap<String, String>(); String curType = null; final String UNKNOWN = "unknown"; final String MIXED = "http://www.w3.org/2001/XMLSchema#string"; // **** Column Names and types. Parallel arrays. Types still unknown. **** for (Object colObj : colNamesJsonArray) { String colStr = (String) colObj; colsForNewTable.add(colStr); colTypesForNewTable.add(UNKNOWN); colNumHash.put(colStr, colsForNewTable.size() - 1); // hash the column indices colTypeHash.put(colStr, UNKNOWN); // hash the column types } // **** Rows **** for (int i = 0; i < rowsJsonArray.size(); i++) { // sample row: {"Test":{"type":"literal","value":"http:\/\/research.ge.com\/dataset#Test_1276"},"testnum":{"datatype":"http:\/\/www.w3.org\/2001\/XMLSchema#integer","type":"typed-literal","value":"1276"}} JSONObject row = (JSONObject) rowsJsonArray.get(i); rowForNewTable = new ArrayList<String>(); // start new row //**** loop through columns in the correct order **** for (Object colObj : colNamesJsonArray) { key = (String) colObj; jsonCell = (JSONObject) row.get(key); if (jsonCell == null) { rowForNewTable.add(""); } else { valueValue = (String) jsonCell.get("value"); valueType = (String) jsonCell.get("type"); valueDataType = (valueType.equals("typed-literal")) ? (String) jsonCell.get("datatype") : valueType; // e.g. "http:\/\/www.w3.org\/2001\/XMLSchema#integer", but only if type is "typed-literal" // add the value to the row rowForNewTable.add(valueValue); // check the type curType = colTypeHash.get(key); // fix UNKNOWN's as they become known if (curType.equals(MIXED)) { // do nothing if cell is already MIXED } else if (curType.equals(UNKNOWN)) { colTypeHash.put(key, valueDataType); colTypesForNewTable.set(colNumHash.get(key), valueDataType); // insert MIXED if types are coming back funny } else if (!curType.equals(valueDataType)) { colTypeHash.put(key, MIXED); colTypesForNewTable.set(colNumHash.get(key), MIXED); } } } rowsForNewTable.add(rowForNewTable); // add the row to the set of rows } String[] colsForNewTableArray = colsForNewTable.toArray(new String[0]); String[] colTypesForNewTableArray = colTypesForNewTable.toArray(new String[0]); // create JSON to return Table table = new Table(colsForNewTableArray, colTypesForNewTableArray, rowsForNewTable); JSONObject tableJson = table.toJson(); return tableJson; } }