com.javafxpert.wikibrowser.WikiVisGraphController.java Source code

Java tutorial

Introduction

Here is the source code for com.javafxpert.wikibrowser.WikiVisGraphController.java

Source

/*
 * Copyright 2015 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.javafxpert.wikibrowser;

import com.javafxpert.wikibrowser.model.conceptmap.*;
import com.javafxpert.wikibrowser.model.locator.ItemInfoResponse;
import com.javafxpert.wikibrowser.model.thumbnail.ThumbnailCache;
import com.javafxpert.wikibrowser.model.thumbnail.ThumbnailResponse;
import com.javafxpert.wikibrowser.model.visgraph.VisGraphEdgeNear;
import com.javafxpert.wikibrowser.model.visgraph.VisGraphNodeNear;
import com.javafxpert.wikibrowser.model.visgraph.VisGraphResponseNear;
import com.javafxpert.wikibrowser.util.WikiBrowserUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.*;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;

import java.util.*;

/**
 * Created by jamesweaver on 10/13/15.
 */
@RestController
public class WikiVisGraphController {
    private Log log = LogFactory.getLog(getClass());

    private final WikiBrowserProperties wikiBrowserProperties;

    @Autowired
    public WikiVisGraphController(WikiBrowserProperties wikiBrowserProperties) {
        this.wikiBrowserProperties = wikiBrowserProperties;
    }

    /**
     * Query Neo4j for all relationships between given set of item IDs
     * @param items
     * @return
     */
    @RequestMapping(value = "/visgraph", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
    public ResponseEntity<Object> search(@RequestParam(value = "items", defaultValue = "") String items) {
        // Example endpoint usage is graph?items=Q24, Q30, Q23, Q16, Q20
        // Scrub the input, and output a string for the Cypher query similar to the following:
        // 'Q24','Q30','Q23','Q16','Q20'
        String argStr = WikiBrowserUtils.scrubItemIds(items, true);

        log.info("argStr=" + argStr);

        VisGraphResponseNear visGraphResponseNear = null;

        if (argStr.length() == 0) {
            // TODO: Consider handling an invalid items argument better than the way it is handled here
            //argStr = "'Q2'"; // If the items argumentisn't valid, pretend Q2 (Earth) was entered
        }

        if (argStr.length() > 0) {
            String neoCypherUrl = wikiBrowserProperties.getNeoCypherUrl();

            /*  Example Cypher query POST
            {
              "statements" : [ {
                "statement" : "MATCH (a)-[r]->(b) WHERE a.itemId IN ['Q24', 'Q30', 'Q23', 'Q16', 'Q20'] AND b.itemId IN ['Q24', 'Q30', 'Q23', 'Q16', 'Q20'] RETURN a, b, r",
                "resultDataContents" : ["graph" ]
              } ]
            }
            */

            /*
            MATCH (a:Item), (b:Item)
            WHERE a.itemId IN ['Q2', 'Q24', 'Q30'] AND b.itemId IN ['Q2', 'Q24', 'Q30']
            WITH a, b
            OPTIONAL MATCH (a)-[rel]-(b)
            RETURN a, b, collect(rel)
                
            was:
            MATCH (a:Item)
            WHERE a.itemId IN ['Q2', 'Q24', 'Q30']
            OPTIONAL MATCH (a:Item)-[rel]-(b:Item)
            WHERE b.itemId IN ['Q2', 'Q24', 'Q30']
            RETURN a, b, collect(rel)
            */

            String qa = "{\"statements\":[{\"statement\":\"MATCH (a:Item), (b:Item) WHERE a.itemId IN [";
            String qb = argStr; // Item IDs
            String qc = "] AND b.itemId IN [";
            String qd = argStr; // Item IDs
            String qe = "] WITH a, b OPTIONAL MATCH (a)-[rel]-(b) RETURN a, b, collect(rel)\",";
            String qf = "\"resultDataContents\":[\"graph\"]}]}";

            /*
            String qa = "{\"statements\":[{\"statement\":\"MATCH (a:Item) WHERE a.itemId IN [";
            String qb = argStr; // Item IDs
            String qc = "] OPTIONAL MATCH (a:Item)-[rel]-(b:Item) WHERE b.itemId IN [";
            String qd = argStr; // Item IDs
            String qe = "] RETURN a, b, collect(rel)\",";
            String qf = "\"resultDataContents\":[\"graph\"]}]}";
            */

            String postString = qa + qb + qc + qd + qe + qf;

            visGraphResponseNear = queryProcessSearchResponse(neoCypherUrl, postString);
        }

        return Optional.ofNullable(visGraphResponseNear).map(cr -> new ResponseEntity<>((Object) cr, HttpStatus.OK))
                .orElse(new ResponseEntity<>("visgraph query unsuccessful", HttpStatus.INTERNAL_SERVER_ERROR));
    }

    /**
     * Retrieve all paths through any properties, with a length of one or two hops, between two given item IDs.
     * TODO: Consider adding a LIMIT parameter
     * @param itemId
     * @param targetId
     * @return
     */
    @RequestMapping(value = "/visshortpaths", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
    public ResponseEntity<Object> retrieveShortPaths(@RequestParam(value = "id", defaultValue = "Q1") String itemId,
            @RequestParam(value = "target", defaultValue = "Q323") String targetId) {
        // Example endpoint usage is shortpaths?id=Q23&target=Q9696

        VisGraphResponseNear visGraphResponseNear = null;

        String neoCypherUrl = wikiBrowserProperties.getNeoCypherUrl();

        /*  Example Cypher query POST
        {
          "statements" : [ {
            "statement" : "MATCH p=allShortestPaths( (a:Item {itemId:'Q23'})-[*..2]-(b:Item {itemId:'Q9696'}) ) RETURN p",
            "resultDataContents" : ["graph" ]
          } ]
        }
        */

        /*
        MATCH p=allShortestPaths( (a:Item {itemId:'Q23'})-[*..2]-(b:Item {itemId:'Q9696'}) )
        RETURN p LIMIT 200
        */

        String qa = "{\"statements\":[{\"statement\":\"MATCH p=allShortestPaths( (a:Item {itemId:'";
        String qb = itemId; // starting item ID
        String qc = "'})-[*..2]-(b:Item {itemId:'";
        String qd = targetId; // target item ID
        String qe = "'}) ) WHERE NONE(x IN NODES(p) WHERE x:Item AND x.itemId = 'Q5') "; // Don't return paths that contain human item ID, or described by relationships
        String qf = "AND NONE(y IN RELATIONSHIPS(p) WHERE y.propId = 'P1343') RETURN p LIMIT 200\",";
        String qg = "\"resultDataContents\":[\"graph\"]}]}";

        String postString = qa + qb + qc + qd + qe + qf + qg;

        visGraphResponseNear = queryProcessSearchResponse(neoCypherUrl, postString);

        return Optional.ofNullable(visGraphResponseNear).map(cr -> new ResponseEntity<>((Object) cr, HttpStatus.OK))
                .orElse(new ResponseEntity<>("visshortpaths query unsuccessful", HttpStatus.INTERNAL_SERVER_ERROR));
    }

    /**
     * Retrieve shortest path from a given item ID to the root of the items (Entity Q35120) that contains only
     * subclass-of (P279), instance-of (P31), and part-of
     * @param itemId
     * @return
     */
    @RequestMapping(value = "/visrootpaths", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
    public ResponseEntity<Object> retrieveShortestPathsToRoot(
            @RequestParam(value = "id", defaultValue = "Q2") String itemId) {
        // Example endpoint usage is rootpath?id=Q319

        String targetId = "Q35120"; // Wikidata Entity item ID

        VisGraphResponseNear visGraphResponseNear = null;

        String neoCypherUrl = wikiBrowserProperties.getNeoCypherUrl();

        /*  Example Cypher query POST
        {
          "statements" : [ {
            "statement" : "MATCH p=allShortestPaths( (a:Item {itemId:'Q319'})-[*]->(b:Item {itemId:'Q35120'}) ) WHERE NONE(x IN RELATIONSHIPS(p) WHERE (x.propId <> 'P279') AND (x.propId <> 'P31') AND (x.propId <> 'P361')) RETURN p",
            "resultDataContents" : ["graph" ]
          } ]
        }
        */

        /*
        MATCH p=allShortestPaths( (a:Item {itemId:'Q319'})-[*]->(b:Item {itemId:'Q35120'}) )
        WHERE NONE(x IN RELATIONSHIPS(p) WHERE (x.propId <> 'P279') AND (x.propId <> 'P31') AND (x.propId <> 'P361'))
        RETURN p
        */

        String qa = "{\"statements\":[{\"statement\":\"MATCH p=allShortestPaths( (a:Item {itemId:'";
        String qb = itemId; // starting item ID
        String qc = "'})-[*]->(b:Item {itemId:'";
        String qd = targetId; // target item ID
        // Regard a path if it contains only subclass-of, instance-of, and part-of relationships
        String qe = "'}) ) WHERE NONE(x IN RELATIONSHIPS(p) WHERE (x.propId <> 'P279') AND (x.propId <> 'P31') AND (x.propId <> 'P361')) ";
        String qf = "RETURN p\",";
        String qg = "\"resultDataContents\":[\"graph\"]}]}";

        String postString = qa + qb + qc + qd + qe + qf + qg;

        visGraphResponseNear = queryProcessSearchResponse(neoCypherUrl, postString);

        return Optional.ofNullable(visGraphResponseNear).map(cr -> new ResponseEntity<>((Object) cr, HttpStatus.OK))
                .orElse(new ResponseEntity<>("visrootpaths query unsuccessful", HttpStatus.INTERNAL_SERVER_ERROR));
    }

    /**
     * Calls the Neo4j Transactional Cypher service and returns an object that holds results
     * @param neoCypherUrl
     * @param postString
     * @return
     */
    private VisGraphResponseNear queryProcessSearchResponse(String neoCypherUrl, String postString) {
        log.info("neoCypherUrl: " + neoCypherUrl);
        log.info("postString: " + postString);

        RestTemplate restTemplate = new RestTemplate();

        HttpHeaders httpHeaders = WikiBrowserUtils.createHeaders(wikiBrowserProperties.getCypherUsername(),
                wikiBrowserProperties.getCypherPassword());

        httpHeaders.setContentType(MediaType.APPLICATION_JSON);
        httpHeaders.setAccept(Arrays.asList(MediaType.APPLICATION_JSON));

        HttpEntity request = new HttpEntity(postString, httpHeaders);

        GraphResponseFar graphResponseFar = null;
        VisGraphResponseNear visGraphResponseNear = new VisGraphResponseNear();
        try {
            ResponseEntity<GraphResponseFar> result = restTemplate.exchange(neoCypherUrl, HttpMethod.POST, request,
                    GraphResponseFar.class);
            graphResponseFar = result.getBody();
            log.info("graphResponseFar: " + graphResponseFar);

            // Populate VisGraphResponseNear instance from GraphResponseFar instance
            HashMap<String, VisGraphNodeNear> visGraphNodeNearMap = new HashMap<>();
            HashMap<String, VisGraphEdgeNear> visGraphEdgeNearMap = new HashMap<>();

            List<ResultFar> resultFarList = graphResponseFar.getResultFarList();
            if (resultFarList.size() > 0) {
                List<DataFar> dataFarList = resultFarList.get(0).getDataFarList();
                Iterator<DataFar> dataFarIterator = dataFarList.iterator();

                while (dataFarIterator.hasNext()) {
                    GraphFar graphFar = dataFarIterator.next().getGraphFar();

                    List<GraphNodeFar> graphNodeFarList = graphFar.getGraphNodeFarList();
                    Iterator<GraphNodeFar> graphNodeFarIterator = graphNodeFarList.iterator();

                    while (graphNodeFarIterator.hasNext()) {
                        GraphNodeFar graphNodeFar = graphNodeFarIterator.next();
                        VisGraphNodeNear visGraphNodeNear = new VisGraphNodeNear();

                        //visGraphNodeNear.setDbId(graphNodeFar.getId());  // Database ID for this node
                        visGraphNodeNear.setDbId(graphNodeFar.getGraphNodePropsFar().getItemId().substring(1));

                        visGraphNodeNear.setTitle(graphNodeFar.getGraphNodePropsFar().getTitle());
                        visGraphNodeNear.setLabelsList(graphNodeFar.getLabelsList());
                        visGraphNodeNear.setItemId(graphNodeFar.getGraphNodePropsFar().getItemId());

                        String itemId = visGraphNodeNear.getItemId();
                        String articleTitle = visGraphNodeNear.getTitle();

                        // Retrieve the article's image
                        String thumbnailUrl = null;

                        String articleLang = "en";
                        // TODO: Add a language property to Item nodes stored in Neo4j that aren't currently in English,
                        //       and use that property to mutate articleTitleLang

                        // First, try to get the thumbnail by ID from cache
                        thumbnailUrl = ThumbnailCache.getThumbnailUrlById(itemId, articleLang);

                        if (thumbnailUrl == null) {
                            // If not available, try to get thumbnail by ID from ThumbnailService
                            try {
                                String thumbnailByIdUrl = this.wikiBrowserProperties
                                        .getThumbnailByIdServiceUrl(itemId, articleLang);
                                thumbnailUrl = new RestTemplate().getForObject(thumbnailByIdUrl, String.class);

                                if (thumbnailUrl != null) {
                                    visGraphNodeNear.setImageUrl(thumbnailUrl);
                                } else {
                                    // If thumbnail isn't available by ID, try to get thumbnail by article title
                                    try {
                                        String thumbnailByTitleUrl = this.wikiBrowserProperties
                                                .getThumbnailByTitleServiceUrl(articleTitle, articleLang);
                                        thumbnailUrl = new RestTemplate().getForObject(thumbnailByTitleUrl,
                                                String.class);

                                        if (thumbnailUrl != null) {
                                            visGraphNodeNear.setImageUrl(thumbnailUrl);
                                        } else {
                                            visGraphNodeNear.setImageUrl("");
                                        }
                                        //log.info("thumbnailUrl:" + thumbnailUrl);
                                    } catch (Exception e) {
                                        e.printStackTrace();
                                        log.info("Caught exception when calling /thumbnail?title=" + articleTitle
                                                + " : " + e);
                                    }
                                }
                                //log.info("thumbnailUrl:" + thumbnailUrl);
                            } catch (Exception e) {
                                e.printStackTrace();
                                log.info("Caught exception when calling /thumbnail?id=" + itemId + " : " + e);
                            }
                        } else {
                            visGraphNodeNear.setImageUrl(thumbnailUrl);
                        }

                        /*
                        // Check cache for thumbnail
                        thumbnailUrl = ThumbnailCache.getThumbnailUrlById(itemId, articleLang);
                            
                        if (thumbnailUrl != null && thumbnailUrl.length() > 0) {
                          // Thumbnail image found in cache by ID, which is the preferred location
                          visGraphNodeNear.setImageUrl(thumbnailUrl);
                        }
                        else {
                          // Thumbnail image not found in cache by ID, so look with Wikimedia API by article title
                          log.info("Thumbnail not found in cache for itemId: " + itemId + ", lang: " + articleLang + " so looking with Wikimedia API by article title");
                            
                          try {
                            String url = this.wikiBrowserProperties.getThumbnailByTitleServiceUrl(articleTitle, articleLang);
                            thumbnailUrl = new RestTemplate().getForObject(url,
                                String.class);
                            
                            if (thumbnailUrl != null && thumbnailUrl.length() > 0) {
                              visGraphNodeNear.setImageUrl(thumbnailUrl);
                            }
                            else {
                              log.info("Thumbnail not found for articleTitle: " + articleTitle + ", trying by itemId: " + itemId);
                            
                              try {
                                String url = this.wikiBrowserProperties.getThumbnailByIdServiceUrl(itemId, articleLang);
                                thumbnailUrl = new RestTemplate().getForObject(url,
                                    String.class);
                            
                                if (thumbnailUrl != null) {
                                  visGraphNodeNear.setImageUrl(thumbnailUrl);
                            
                                  // Because successful, cache by article title
                                  ThumbnailCache.setThumbnailUrlByTitle(articleTitle, articleLang, thumbnailUrl);
                                }
                                else {
                                  visGraphNodeNear.setImageUrl("");
                                }
                                //log.info("thumbnailUrl:" + thumbnailUrl);
                              }
                              catch (Exception e) {
                                e.printStackTrace();
                                log.info("Caught exception when calling /thumbnail?id=" + itemId + " : " + e);
                              }
                            
                            }
                            //log.info("thumbnailUrl:" + thumbnailUrl);
                          } catch (Exception e) {
                            e.printStackTrace();
                            log.info("Caught exception when calling /thumbnail?title=" + articleTitle + " : " + e);
                          }
                        }
                        */

                        // Note: The key in the graphNodeNearMap is the Neo4j node id, not the Wikidata item ID
                        visGraphNodeNearMap.put(graphNodeFar.getId(), visGraphNodeNear);
                    }

                    List<GraphRelationFar> graphRelationFarList = graphFar.getGraphRelationFarList();
                    Iterator<GraphRelationFar> graphRelationFarIterator = graphRelationFarList.iterator();

                    while (graphRelationFarIterator.hasNext()) {
                        GraphRelationFar graphRelationFar = graphRelationFarIterator.next();
                        VisGraphEdgeNear visGraphEdgeNear = new VisGraphEdgeNear();

                        // Use the Neo4j node ids from the relationship to retrieve the Wikidata Item IDs from the graphNodeNearMap
                        String neo4jStartNodeId = graphRelationFar.getStartNode();
                        String wikidataStartNodeItemId = visGraphNodeNearMap.get(neo4jStartNodeId).getItemId();
                        String neo4jEndNodeId = graphRelationFar.getEndNode();
                        String wikidataEndNodeItemId = visGraphNodeNearMap.get(neo4jEndNodeId).getItemId();

                        //visGraphEdgeNear.setFromDbId(neo4jStartNodeId);
                        visGraphEdgeNear.setFromDbId(wikidataStartNodeItemId.substring(1));

                        //visGraphEdgeNear.setToDbId(neo4jEndNodeId);
                        visGraphEdgeNear.setToDbId(wikidataEndNodeItemId.substring(1));

                        visGraphEdgeNear.setLabel(graphRelationFar.getType());
                        visGraphEdgeNear.setArrowDirection("to");
                        visGraphEdgeNear.setPropId(graphRelationFar.getGraphRelationPropsFar().getPropId());

                        visGraphEdgeNear.setFromItemId(wikidataStartNodeItemId);
                        visGraphEdgeNear.setToItemId(wikidataEndNodeItemId);

                        // Note: The key in the graphLinkNearMap is the Neo4j node id, not the Wikidata item ID
                        visGraphEdgeNearMap.put(graphRelationFar.getId(), visGraphEdgeNear);
                    }
                }

                // Create and populate a List of nodes to set into the graphResponseNear instance
                List<VisGraphNodeNear> visGraphNodeNearList = new ArrayList<>();
                visGraphNodeNearMap.forEach((k, v) -> {
                    visGraphNodeNearList.add(v);
                });
                visGraphResponseNear.setVisGraphNodeNearList(visGraphNodeNearList);

                // Create and populate a List of links to set into the graphResponseNear instance
                List<VisGraphEdgeNear> visGraphEdgeNearList = new ArrayList<>();
                visGraphEdgeNearMap.forEach((k, v) -> {
                    visGraphEdgeNearList.add(v);
                });
                visGraphResponseNear.setVisGraphEdgeNearList(visGraphEdgeNearList);
            }
        } catch (Exception e) {
            e.printStackTrace();
            log.info("Caught exception when calling Neo Cypher service " + e);
        }

        return visGraphResponseNear;
    }
}