Java tutorial
/******************************************************************************* * Copyright 2012 University of Southern California * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * This code was developed by the Information Integration Group as part * of the Karma project at the Information Sciences Institute of the * University of Southern California. For more information, publications, * and related projects, please see: http://www.isi.edu/integration ******************************************************************************/ package edu.isi.karma.research.modeling; import java.io.File; import java.io.PrintWriter; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.jgrapht.graph.AsUndirectedGraph; import org.jgrapht.graph.DirectedWeightedMultigraph; import org.jgrapht.graph.WeightedMultigraph; import org.python.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import edu.isi.karma.config.ModelingConfiguration; import edu.isi.karma.config.ModelingConfigurationRegistry; import edu.isi.karma.er.helper.PythonRepository; import edu.isi.karma.er.helper.PythonRepositoryRegistry; import edu.isi.karma.modeling.alignment.GraphBuilder; import edu.isi.karma.modeling.alignment.GraphBuilderTopK; import edu.isi.karma.modeling.alignment.GraphUtil; import edu.isi.karma.modeling.alignment.GraphVizLabelType; import edu.isi.karma.modeling.alignment.GraphVizUtil; import edu.isi.karma.modeling.alignment.LinkIdFactory; import edu.isi.karma.modeling.alignment.ModelEvaluation; import edu.isi.karma.modeling.alignment.NodeIdFactory; import edu.isi.karma.modeling.alignment.SemanticModel; import edu.isi.karma.modeling.alignment.SteinerTree; import edu.isi.karma.modeling.alignment.TreePostProcess; import edu.isi.karma.modeling.alignment.learner.CandidateSteinerSets; import edu.isi.karma.modeling.alignment.learner.ModelLearningGraph; import edu.isi.karma.modeling.alignment.learner.ModelLearningGraphCompact; import edu.isi.karma.modeling.alignment.learner.ModelLearningGraphType; import edu.isi.karma.modeling.alignment.learner.ModelReader; import edu.isi.karma.modeling.alignment.learner.PatternWeightSystem; import edu.isi.karma.modeling.alignment.learner.SemanticTypeMapping; import edu.isi.karma.modeling.alignment.learner.SortableSemanticModel; import edu.isi.karma.modeling.alignment.learner.SteinerNodes; import edu.isi.karma.modeling.ontology.OntologyManager; import edu.isi.karma.modeling.research.Params; import edu.isi.karma.rep.alignment.ClassInstanceLink; import edu.isi.karma.rep.alignment.ColumnNode; import edu.isi.karma.rep.alignment.ColumnSemanticTypeStatus; import edu.isi.karma.rep.alignment.DataPropertyLink; import edu.isi.karma.rep.alignment.DefaultLink; import edu.isi.karma.rep.alignment.InternalNode; import edu.isi.karma.rep.alignment.Label; import edu.isi.karma.rep.alignment.LabeledLink; import edu.isi.karma.rep.alignment.Node; import edu.isi.karma.rep.alignment.SemanticType; import edu.isi.karma.rep.alignment.SemanticType.Origin; import edu.isi.karma.semantictypes.evaluation.EvaluateMRR; import edu.isi.karma.semantictypes.evaluation.MRRItem; import edu.isi.karma.util.RandomGUID; import edu.isi.karma.webserver.ContextParametersRegistry; import edu.isi.karma.webserver.ServletContextParameterMap; import edu.isi.karma.webserver.ServletContextParameterMap.ContextParameter; public class ModelLearner_KnownModels { private static Logger logger = LoggerFactory.getLogger(ModelLearner_KnownModels.class); private OntologyManager ontologyManager = null; private GraphBuilder graphBuilder = null; private NodeIdFactory nodeIdFactory = null; private List<Node> steinerNodes = null; public ModelLearner_KnownModels(OntologyManager ontologyManager, List<Node> steinerNodes) { if (ontologyManager == null || steinerNodes == null || steinerNodes.isEmpty()) { logger.error("cannot instanciate model learner!"); return; } GraphBuilder gb = ModelLearningGraph.getInstance(ontologyManager, ModelLearningGraphType.Compact) .getGraphBuilder(); this.ontologyManager = ontologyManager; this.steinerNodes = steinerNodes; // if (this.steinerNodes != null) Collections.sort(this.steinerNodes); this.graphBuilder = cloneGraphBuilder(gb); // create a copy of the graph builder this.nodeIdFactory = this.graphBuilder.getNodeIdFactory(); } public ModelLearner_KnownModels(GraphBuilder graphBuilder, List<Node> steinerNodes) { if (graphBuilder == null || steinerNodes == null || steinerNodes.isEmpty()) { logger.error("cannot instanciate model learner!"); return; } this.ontologyManager = graphBuilder.getOntologyManager(); this.steinerNodes = steinerNodes; // if (this.steinerNodes != null) Collections.sort(this.steinerNodes); this.graphBuilder = cloneGraphBuilder(graphBuilder); // create a copy of the graph builder this.nodeIdFactory = this.graphBuilder.getNodeIdFactory(); } private GraphBuilder cloneGraphBuilder(GraphBuilder graphBuilder) { GraphBuilder clonedGraphBuilder = null; if (graphBuilder == null || graphBuilder.getGraph() == null) { clonedGraphBuilder = new GraphBuilderTopK(this.ontologyManager, false); } else { clonedGraphBuilder = new GraphBuilderTopK(this.ontologyManager, graphBuilder.getGraph()); } return clonedGraphBuilder; } public List<SortableSemanticModel> hypothesize(boolean useCorrectTypes, int numberOfCandidates) throws Exception { ModelingConfiguration modelingConfiguration = ModelingConfigurationRegistry.getInstance() .getModelingConfiguration(ontologyManager.getContextId()); List<SortableSemanticModel> sortableSemanticModels = new ArrayList<SortableSemanticModel>(); Set<Node> addedNodes = new HashSet<Node>(); //They should be deleted from the graph after computing the semantic models List<ColumnNode> columnNodes = new LinkedList<ColumnNode>(); for (Node n : steinerNodes) if (n instanceof ColumnNode) columnNodes.add((ColumnNode) n); // long start = System.currentTimeMillis(); logger.info("finding candidate steiner sets ... "); CandidateSteinerSets candidateSteinerSets = getCandidateSteinerSets(steinerNodes, useCorrectTypes, numberOfCandidates, addedNodes); // long elapsedTimeMillis = System.currentTimeMillis() - start; // float elapsedTimeSec = elapsedTimeMillis/1000F; // System.out.println(elapsedTimeSec); if (candidateSteinerSets == null || candidateSteinerSets.getSteinerSets() == null || candidateSteinerSets.getSteinerSets().isEmpty()) { logger.error("there is no candidate set of steiner nodes."); DirectedWeightedMultigraph<Node, LabeledLink> tree = new DirectedWeightedMultigraph<Node, LabeledLink>( LabeledLink.class); for (Node n : steinerNodes) tree.addVertex(n); SemanticModel sm = new SemanticModel(new RandomGUID().toString(), tree); SortableSemanticModel sortableSemanticModel = new SortableSemanticModel(sm, null, false); sortableSemanticModels.add(sortableSemanticModel); return sortableSemanticModels; } logger.info("graph nodes: " + this.graphBuilder.getGraph().vertexSet().size()); logger.info("graph links: " + this.graphBuilder.getGraph().edgeSet().size()); logger.info("number of steiner sets: " + candidateSteinerSets.numberOfCandidateSets()); // logger.info("updating weights according to training data ..."); // long start = System.currentTimeMillis(); // this.updateWeights(); // long updateWightsElapsedTimeMillis = System.currentTimeMillis() - start; // logger.info("time to update weights: " + (updateWightsElapsedTimeMillis/1000F)); logger.info("computing steiner trees ..."); int number = 0; for (SteinerNodes sn : candidateSteinerSets.getSteinerSets()) { if (sn == null) continue; logger.info("computing steiner tree for steiner nodes set " + number + " ..."); // logger.info(sn.getScoreDetailsString()); // for (Entry<ColumnNode, SemanticTypeMapping> n : sn.getColumnNodeInfo().entrySet()) { // System.out.println(sn.getMappingToSourceColumns().get(n.getKey()).getColumnName() + "---" + // n.getValue().getLink().getId()); // } number++; // logger.info("START ..."); List<DirectedWeightedMultigraph<Node, LabeledLink>> topKSteinerTrees; if (this.graphBuilder instanceof GraphBuilderTopK) { topKSteinerTrees = ((GraphBuilderTopK) this.graphBuilder).getTopKSteinerTrees(sn, modelingConfiguration.getTopKSteinerTree(), 5, 3, true); } else { topKSteinerTrees = new LinkedList<DirectedWeightedMultigraph<Node, LabeledLink>>(); SteinerTree steinerTree = new SteinerTree( new AsUndirectedGraph<Node, DefaultLink>(this.graphBuilder.getGraph()), Lists.newLinkedList(sn.getNodes())); WeightedMultigraph<Node, DefaultLink> t = steinerTree.getDefaultSteinerTree(); TreePostProcess treePostProcess = new TreePostProcess(this.graphBuilder, t); if (treePostProcess.getTree() != null) topKSteinerTrees.add(treePostProcess.getTree()); } // System.out.println(GraphUtil.labeledGraphToString(treePostProcess.getTree())); // logger.info("END ..."); for (DirectedWeightedMultigraph<Node, LabeledLink> tree : topKSteinerTrees) { if (tree != null) { // System.out.println(); SemanticModel sm = new SemanticModel(new RandomGUID().toString(), tree, columnNodes, sn.getMappingToSourceColumns()); SortableSemanticModel sortableSemanticModel = new SortableSemanticModel(sm, sn, false); sortableSemanticModels.add(sortableSemanticModel); // System.out.println(GraphUtil.labeledGraphToString(sm.getGraph())); // System.out.println(sortableSemanticModel.getLinkCoherence().printCoherenceList()); } } if (number >= modelingConfiguration.getNumCandidateMappings()) break; } Collections.sort(sortableSemanticModels); // int count = Math.min(sortableSemanticModels.size(), modelingConfiguration.getNumCandidateMappings()); logger.info("results are ready ..."); // sortableSemanticModels.get(0).print(); // return sortableSemanticModels.subList(0, count); List<SortableSemanticModel> uniqueModels = new ArrayList<SortableSemanticModel>(); SortableSemanticModel current, previous; if (sortableSemanticModels != null) { if (sortableSemanticModels.size() > 0) uniqueModels.add(sortableSemanticModels.get(0)); for (int i = 1; i < sortableSemanticModels.size(); i++) { current = sortableSemanticModels.get(i); previous = sortableSemanticModels.get(i - 1); if (current.getScore() == previous.getScore() && current.getCost() == previous.getCost()) continue; uniqueModels.add(current); } } logger.info("results are ready ..."); return uniqueModels; } // private DirectedWeightedMultigraph<Node, LabeledLink> computeSteinerTree(Set<Node> steinerNodes) { // // if (steinerNodes == null || steinerNodes.size() == 0) { // logger.error("There is no steiner node."); // return null; // } // // // System.out.println(steinerNodes.size()); // List<Node> steinerNodeList = new ArrayList<Node>(steinerNodes); // // long start = System.currentTimeMillis(); // UndirectedGraph<Node, DefaultLink> undirectedGraph = new AsUndirectedGraph<Node, DefaultLink>(this.graphBuilder.getGraph()); // // logger.debug("computing steiner tree ..."); // SteinerTree steinerTree = new SteinerTree(undirectedGraph, steinerNodeList); // DirectedWeightedMultigraph<Node, LabeledLink> tree = new TreePostProcess(this.graphBuilder, steinerTree.getDefaultSteinerTree(), null, false).getTree(); // //(DirectedWeightedMultigraph<Node, LabeledLink>)GraphUtil.asDirectedGraph(steinerTree.getDefaultSteinerTree()); // // logger.debug(GraphUtil.labeledGraphToString(tree)); // // long steinerTreeElapsedTimeMillis = System.currentTimeMillis() - start; // logger.debug("total number of nodes in steiner tree: " + tree.vertexSet().size()); // logger.debug("total number of edges in steiner tree: " + tree.edgeSet().size()); // logger.debug("time to compute steiner tree: " + (steinerTreeElapsedTimeMillis/1000F)); // // return tree; // // // long finalTreeElapsedTimeMillis = System.currentTimeMillis() - steinerTreeElapsedTimeMillis; // // DirectedWeightedMultigraph<Node, Link> finalTree = buildOutputTree(tree); // // logger.info("time to build final tree: " + (finalTreeElapsedTimeMillis/1000F)); // // // GraphUtil.printGraph(finalTree); // // return finalTree; // // } private CandidateSteinerSets getCandidateSteinerSets(List<Node> steinerNodes, boolean useCorrectTypes, int numberOfCandidates, Set<Node> addedNodes) { if (steinerNodes == null || steinerNodes.isEmpty()) return null; int maxNumberOfSteinerNodes = steinerNodes.size() * 2; CandidateSteinerSets candidateSteinerSets = new CandidateSteinerSets(maxNumberOfSteinerNodes, ontologyManager.getContextId()); if (addedNodes == null) addedNodes = new HashSet<Node>(); Set<SemanticTypeMapping> tempSemanticTypeMappings; HashMap<ColumnNode, List<SemanticType>> columnSemanticTypes = new HashMap<ColumnNode, List<SemanticType>>(); HashMap<String, Integer> semanticTypesCount = new HashMap<String, Integer>(); List<SemanticType> candidateSemanticTypes = null; String domainUri = "", propertyUri = ""; for (Node n : steinerNodes) { ColumnNode cn = null; if (n instanceof ColumnNode) cn = (ColumnNode) n; else continue; if (!useCorrectTypes) { candidateSemanticTypes = cn.getTopKLearnedSemanticTypes(numberOfCandidates); } else if (cn.getSemanticTypeStatus() == ColumnSemanticTypeStatus.UserAssigned) { candidateSemanticTypes = cn.getUserSemanticTypes(); } if (candidateSemanticTypes == null) { logger.error("No candidate semantic type found for the column " + cn.getColumnName()); return null; } columnSemanticTypes.put(cn, candidateSemanticTypes); for (SemanticType semanticType : candidateSemanticTypes) { if (semanticType == null || semanticType.getDomain() == null || semanticType.getType() == null) continue; domainUri = semanticType.getDomain().getUri(); propertyUri = semanticType.getType().getUri(); Integer count = semanticTypesCount.get(domainUri + propertyUri); if (count == null) semanticTypesCount.put(domainUri + propertyUri, 1); else semanticTypesCount.put(domainUri + propertyUri, count.intValue() + 1); } } long numOfMappings = 1; for (Node n : steinerNodes) { if (n instanceof InternalNode) continue; ColumnNode cn = null; if (n instanceof ColumnNode) cn = (ColumnNode) n; else continue; candidateSemanticTypes = columnSemanticTypes.get(n); if (candidateSemanticTypes == null) continue; logger.info("===== Column: " + cn.getColumnName()); Set<SemanticTypeMapping> semanticTypeMappings = null; // if (cn.hasUserType()) { // HashMap<SemanticType, LabeledLink> domainLinks = // GraphUtil.getDomainLinks(this.graphBuilder.getGraph(), cn, cn.getUserSemanticTypes()); // if (domainLinks != null && !domainLinks.isEmpty()) { // for (SemanticType st : cn.getUserSemanticTypes()) { // semanticTypeMappings = new HashSet<SemanticTypeMapping>(); // LabeledLink domainLink = domainLinks.get(st); // if (domainLink == null || domainLink.getSource() == null || !(domainLink.getSource() instanceof InternalNode)) // continue; // SemanticTypeMapping mp = // new SemanticTypeMapping(cn, st, (InternalNode)domainLink.getSource(), domainLink, cn); // semanticTypeMappings.add(mp); // candidateSteinerSets.updateSteinerSets(semanticTypeMappings); // } // } // } else { semanticTypeMappings = new HashSet<SemanticTypeMapping>(); for (SemanticType semanticType : candidateSemanticTypes) { logger.info( "\t" + semanticType.getConfidenceScore() + " :" + semanticType.getModelLabelString()); if (semanticType == null || semanticType.getDomain() == null || semanticType.getType() == null) continue; domainUri = semanticType.getDomain().getUri(); propertyUri = semanticType.getType().getUri(); Integer countOfSemanticType = semanticTypesCount.get(domainUri + propertyUri); logger.debug("count of semantic type: " + countOfSemanticType); tempSemanticTypeMappings = findSemanticTypeInGraph(cn, semanticType, semanticTypesCount, addedNodes); logger.debug("number of matches for semantic type: " + +(tempSemanticTypeMappings == null ? 0 : tempSemanticTypeMappings.size())); if (tempSemanticTypeMappings != null) semanticTypeMappings.addAll(tempSemanticTypeMappings); int countOfMatches = tempSemanticTypeMappings == null ? 0 : tempSemanticTypeMappings.size(); // if (countOfMatches < countOfSemanticType) if (countOfMatches == 0) // No struct in graph is matched with the semantic type, we add a new struct to the graph { SemanticTypeMapping mp = addSemanticTypeStruct(cn, semanticType, addedNodes); if (mp != null) semanticTypeMappings.add(mp); } } // System.out.println("number of matches for column " + n.getColumnName() + // ": " + (semanticTypeMappings == null ? 0 : semanticTypeMappings.size())); logger.debug("number of matches for column " + cn.getColumnName() + ": " + (semanticTypeMappings == null ? 0 : semanticTypeMappings.size())); numOfMappings *= (semanticTypeMappings == null || semanticTypeMappings.isEmpty() ? 1 : semanticTypeMappings.size()); logger.debug("number of candidate steiner sets before update: " + candidateSteinerSets.getSteinerSets().size()); candidateSteinerSets.updateSteinerSets(semanticTypeMappings); logger.debug("number of candidate steiner sets after update: " + candidateSteinerSets.getSteinerSets().size()); } } for (Node n : steinerNodes) { if (n instanceof InternalNode) { candidateSteinerSets.updateSteinerSets((InternalNode) n); } } // System.out.println("number of possible mappings: " + numOfMappings); logger.info("number of possible mappings: " + numOfMappings); return candidateSteinerSets; } private Set<SemanticTypeMapping> findSemanticTypeInGraph(ColumnNode sourceColumn, SemanticType semanticType, HashMap<String, Integer> semanticTypesCount, Set<Node> addedNodes) { logger.debug("finding matches for semantic type in the graph ... "); ModelingConfiguration modelingConfiguration = ModelingConfigurationRegistry.getInstance() .getModelingConfiguration(ontologyManager.getContextId()); if (addedNodes == null) addedNodes = new HashSet<Node>(); Set<SemanticTypeMapping> mappings = new HashSet<SemanticTypeMapping>(); if (semanticType == null) { logger.error("semantic type is null."); return mappings; } if (semanticType.getDomain() == null) { logger.error("semantic type does not have any domain"); return mappings; } if (semanticType.getType() == null) { logger.error("semantic type does not have any link"); return mappings; } String domainUri = semanticType.getDomain().getUri(); String propertyUri = semanticType.getType().getUri(); Double confidence = semanticType.getConfidenceScore(); Origin origin = semanticType.getOrigin(); Integer countOfSemanticType = semanticTypesCount.get(domainUri + propertyUri); if (countOfSemanticType == null) { logger.error("count of semantic type should not be null or zero"); return mappings; } if (domainUri == null || domainUri.isEmpty()) { logger.error("semantic type does not have any domain"); return mappings; } if (propertyUri == null || propertyUri.isEmpty()) { logger.error("semantic type does not have any link"); return mappings; } logger.debug("semantic type: " + domainUri + "|" + propertyUri + "|" + confidence + "|" + origin); // add dataproperty to existing classes if sl is a data node mapping // Set<Node> foundInternalNodes = new HashSet<Node>(); Set<SemanticTypeMapping> semanticTypeMatches = this.graphBuilder.getSemanticTypeMatches() .get(domainUri + propertyUri); if (semanticTypeMatches != null) { for (SemanticTypeMapping stm : semanticTypeMatches) { SemanticTypeMapping mp = new SemanticTypeMapping(sourceColumn, semanticType, stm.getSource(), stm.getLink(), stm.getTarget()); mappings.add(mp); // foundInternalNodes.add(stm.getSource()); } } logger.debug("adding data property to the found internal nodes ..."); Integer count; boolean allowMultipleSamePropertiesPerNode = modelingConfiguration.isMultipleSamePropertyPerNode(); Set<Node> nodesWithSameUriOfDomain = this.graphBuilder.getUriToNodesMap().get(domainUri); if (nodesWithSameUriOfDomain != null) { for (Node source : nodesWithSameUriOfDomain) { count = this.graphBuilder.getNodeDataPropertyCount().get(source.getId() + propertyUri); if (count != null) { if (allowMultipleSamePropertiesPerNode) { if (count >= countOfSemanticType.intValue()) continue; } else { if (count >= 1) continue; } } String nodeId = new RandomGUID().toString(); ColumnNode target = new ColumnNode(nodeId, nodeId, sourceColumn.getColumnName(), null); if (!this.graphBuilder.addNode(target)) continue; ; addedNodes.add(target); String linkId = LinkIdFactory.getLinkId(propertyUri, source.getId(), target.getId()); LabeledLink link = new DataPropertyLink(linkId, new Label(propertyUri)); if (!this.graphBuilder.addLink(source, target, link)) continue; ; SemanticTypeMapping mp = new SemanticTypeMapping(sourceColumn, semanticType, (InternalNode) source, link, target); mappings.add(mp); } } return mappings; } private SemanticTypeMapping addSemanticTypeStruct(ColumnNode sourceColumn, SemanticType semanticType, Set<Node> addedNodes) { logger.debug("adding semantic type to the graph ... "); if (addedNodes == null) addedNodes = new HashSet<Node>(); if (semanticType == null) { logger.error("semantic type is null."); return null; } if (semanticType.getDomain() == null) { logger.error("semantic type does not have any domain"); return null; } if (semanticType.getType() == null) { logger.error("semantic type does not have any link"); return null; } String domainUri = semanticType.getDomain().getUri(); String propertyUri = semanticType.getType().getUri(); Double confidence = semanticType.getConfidenceScore(); Origin origin = semanticType.getOrigin(); if (domainUri == null || domainUri.isEmpty()) { logger.error("semantic type does not have any domain"); return null; } if (propertyUri == null || propertyUri.isEmpty()) { logger.error("semantic type does not have any link"); return null; } logger.debug("semantic type: " + domainUri + "|" + propertyUri + "|" + confidence + "|" + origin); InternalNode source = null; String nodeId; nodeId = nodeIdFactory.getNodeId(domainUri); source = new InternalNode(nodeId, new Label(domainUri)); if (!this.graphBuilder.addNodeAndUpdate(source, addedNodes)) return null; nodeId = new RandomGUID().toString(); ColumnNode target = new ColumnNode(nodeId, nodeId, sourceColumn.getColumnName(), null); if (!this.graphBuilder.addNode(target)) return null; addedNodes.add(target); String linkId = LinkIdFactory.getLinkId(propertyUri, source.getId(), target.getId()); LabeledLink link; if (propertyUri.equalsIgnoreCase(ClassInstanceLink.getFixedLabel().getUri())) link = new ClassInstanceLink(linkId); else { Label label = this.ontologyManager.getUriLabel(propertyUri); link = new DataPropertyLink(linkId, label); } if (!this.graphBuilder.addLink(source, target, link)) return null; SemanticTypeMapping mappingStruct = new SemanticTypeMapping(sourceColumn, semanticType, source, link, target); return mappingStruct; } private static double roundDecimals(double d, int k) { String format = ""; for (int i = 0; i < k; i++) format += "#"; DecimalFormat DForm = new DecimalFormat("#." + format); return Double.valueOf(DForm.format(d)); } private static void getStatistics(List<SemanticModel> semanticModels, List<String> modelFiles, int numOfCandidates) { for (int i = 0; i < semanticModels.size(); i++) { SemanticModel source = semanticModels.get(i); int attributeCount = source.getColumnNodes().size(); int nodeCount = source.getGraph().vertexSet().size(); int linkCount = source.getGraph().edgeSet().size(); int datanodeCount = 0; int classNodeCount = 0; for (Node n : source.getGraph().vertexSet()) { if (n instanceof InternalNode) classNodeCount++; if (n instanceof ColumnNode) datanodeCount++; } // System.out.println(attributeCount + "\t" + nodeCount + "\t" + linkCount + "\t" + classNodeCount + "\t" + datanodeCount); List<ColumnNode> columnNodes = source.getColumnNodes(); if (columnNodes == null) return; int numberOfAttributesWhoseTypeIsFirstCRFType = 0; int numberOfAttributesWhoseTypeIsInCRFTypes = 0; for (ColumnNode cn : columnNodes) { List<SemanticType> userSemanticTypes = cn.getUserSemanticTypes(); List<SemanticType> top4Suggestions = cn.getTopKLearnedSemanticTypes(4); for (int j = 0; j < top4Suggestions.size(); j++) { SemanticType st = top4Suggestions.get(j); if (userSemanticTypes != null) { for (SemanticType t : userSemanticTypes) { if (st.getModelLabelString().equalsIgnoreCase(t.getModelLabelString())) { if (j == 0) numberOfAttributesWhoseTypeIsFirstCRFType++; numberOfAttributesWhoseTypeIsInCRFTypes++; j = top4Suggestions.size(); break; } } } } } MRRItem mrrItem = null; if (modelFiles != null) { mrrItem = EvaluateMRR.calculateMRRValue(modelFiles.get(i), 4); } // System.out.println(numberOfAttributesWhoseTypeIsInCRFTypes + "\t" + numberOfAttributesWhoseTypeIsFirstCRFType); // System.out.println(source.getName()); System.out.println(attributeCount + "\t" + nodeCount + "\t" + linkCount + "\t" + (linkCount - attributeCount) + "\t" + classNodeCount + "\t" + datanodeCount + "\t" + numberOfAttributesWhoseTypeIsFirstCRFType + "\t" + numberOfAttributesWhoseTypeIsInCRFTypes + "\t" + (mrrItem != null ? roundDecimals(mrrItem.getAccuracy(), 2) : 0) + "\t" + (mrrItem != null ? roundDecimals(mrrItem.getMrr(), 2) : 0)); } } public static void runResearchEvaluation() throws Exception { /*** * When running with k=1, change the flag "multiple.same.property.per.node" to true so all attributes have at least one semantic types */ ServletContextParameterMap contextParameters = ContextParametersRegistry.getInstance() .registerByKarmaHome("/Users/mohsen/karma/"); contextParameters.setParameterValue(ContextParameter.USER_DIRECTORY_PATH, "/Users/mohsen/karma/"); contextParameters.setParameterValue(ContextParameter.USER_CONFIG_DIRECTORY, "/Users/mohsen/karma/config"); contextParameters.setParameterValue(ContextParameter.TRAINING_EXAMPLE_MAX_COUNT, "1000000"); contextParameters.setParameterValue(ContextParameter.SEMTYPE_MODEL_DIRECTORY, "/Users/mohsen/karma/semantic-type-files/"); contextParameters.setParameterValue(ContextParameter.JSON_MODELS_DIR, "/Users/mohsen/karma/models-json/"); contextParameters.setParameterValue(ContextParameter.GRAPHVIZ_MODELS_DIR, "/Users/mohsen/karma/models-graphviz/"); contextParameters.setParameterValue(ContextParameter.USER_PYTHON_SCRIPTS_DIRECTORY, "/Users/mohsen/karma/python/"); contextParameters.setParameterValue(ContextParameter.EVALUATE_MRR, "/Users/mohsen/karma/evaluate-mrr/"); PythonRepository pythonRepository = new PythonRepository(true, contextParameters.getParameterValue(ContextParameter.USER_PYTHON_SCRIPTS_DIRECTORY)); PythonRepositoryRegistry.getInstance().register(pythonRepository); // String inputPath = Params.INPUT_DIR; String graphPath = Params.GRAPHS_DIR; File semFilesFolder = new File( contextParameters.getParameterValue(ContextParameter.SEMTYPE_MODEL_DIRECTORY)); // List<SemanticModel> semanticModels = ModelReader.importSemanticModels(inputPath); List<SemanticModel> semanticModels = ModelReader.importSemanticModelsFromJsonFiles(Params.MODEL_DIR, Params.MODEL_MAIN_FILE_EXT); File[] sources = new File(Params.SOURCE_DIR).listFiles(); File[] r2rmlModels = new File(Params.R2RML_DIR).listFiles(); if (sources.length > 0 && sources[0].getName().startsWith(".")) sources = (File[]) ArrayUtils.removeElement(sources, sources[0]); if (r2rmlModels.length > 0 && r2rmlModels[0].getName().startsWith(".")) r2rmlModels = (File[]) ArrayUtils.removeElement(r2rmlModels, r2rmlModels[0]); List<SemanticModel> trainingData = new ArrayList<SemanticModel>(); File[] trainingSources; File[] trainingModels; File trainingSource = null; File trainingModel = null; File testSource; File testModel; OntologyManager ontologyManager = new OntologyManager(contextParameters.getId()); File ff = new File(Params.ONTOLOGY_DIR); File[] files = ff.listFiles(); for (File f : files) { if (f.getName().startsWith(".") || f.isDirectory()) { continue; //Ignore . files } ontologyManager.doImport(f, "UTF-8"); } ontologyManager.updateCache(); ModelLearningGraph modelLearningGraph = null; ModelLearner_KnownModels modelLearner; boolean onlyGenerateSemanticTypeStatistics = false; boolean iterativeEvaluation = true; boolean useCorrectType = false; boolean onlyEvaluateInternalLinks = false || useCorrectType; boolean zeroKnownModel = false; int numberOfCandidates = 1; if (onlyGenerateSemanticTypeStatistics) { getStatistics(semanticModels, null, 0); return; } int numberOfKnownModels; String filePath = Params.RESULTS_DIR + "temp/"; String filename = ""; filename += "results"; filename += useCorrectType ? "-correct" : "-k=" + numberOfCandidates; filename += zeroKnownModel ? "-ontology" : ""; filename += onlyEvaluateInternalLinks ? "-internal" : "-all"; filename += iterativeEvaluation ? "-iterative" : ""; filename += ".csv"; PrintWriter resultFileIterative = null; PrintWriter resultFile = null; StringBuffer[] resultsArray = null; if (iterativeEvaluation) { resultFileIterative = new PrintWriter(new File(filePath + filename)); resultsArray = new StringBuffer[semanticModels.size() + 2]; for (int i = 0; i < resultsArray.length; i++) { resultsArray[i] = new StringBuffer(); } } else { resultFile = new PrintWriter(new File(filePath + filename)); resultFile.println("source \t p \t r \t t \t a \t m \n"); } // new OfflineTraining().getCorrectModel(contextParameters, // null, null, // sources[20], r2rmlModels[20], 0, numberOfCandidates); // if (true) return; for (int i = 0; i < semanticModels.size(); i++) { // for (int i = 0; i <= 1; i++) { // int i = 1; { // clean semantic files folder in karma home FileUtils.cleanDirectory(semFilesFolder); trainingSource = null; trainingModel = null; int newSourceIndex = i; SemanticModel newSource = semanticModels.get(newSourceIndex); logger.info("======================================================"); logger.info(newSource.getName() + "(#attributes:" + newSource.getColumnNodes().size() + ")"); System.out.println(newSource.getName() + "(#attributes:" + newSource.getColumnNodes().size() + ")"); logger.info("======================================================"); if (zeroKnownModel) numberOfKnownModels = 0; else numberOfKnownModels = iterativeEvaluation ? 0 : semanticModels.size() - 1; if (iterativeEvaluation) { if (resultsArray[0].length() > 0) resultsArray[0].append(" \t "); resultsArray[0].append(newSource.getName() + "(" + newSource.getColumnNodes().size() + ")" + "\t" + " " + "\t" + " " + "\t" + " " + "\t" + " "); if (resultsArray[1].length() > 0) resultsArray[1].append(" \t "); resultsArray[1].append("p \t r \t t \t a \t m"); } // numberOfKnownModels = 2; while (numberOfKnownModels <= semanticModels.size() - 1) { trainingData.clear(); trainingSources = new File[numberOfKnownModels]; trainingModels = new File[numberOfKnownModels]; int j = 0, count = 0; while (count < numberOfKnownModels) { if (j != newSourceIndex) { trainingData.add(semanticModels.get(j)); trainingSources[count] = sources[j]; trainingModels[count] = r2rmlModels[j]; count++; if (count == numberOfKnownModels) { trainingSource = sources[j]; trainingModel = r2rmlModels[j]; } } j++; } modelLearningGraph = (ModelLearningGraphCompact) ModelLearningGraph .getEmptyInstance(ontologyManager, ModelLearningGraphType.Compact); SemanticModel correctModel; if (useCorrectType) { correctModel = newSource; correctModel.setAccuracy(1.0); correctModel.setMrr(1.0); } else { testSource = sources[newSourceIndex]; testModel = r2rmlModels[newSourceIndex]; if (iterativeEvaluation) { correctModel = new OfflineTraining().getCorrectModel(contextParameters, trainingSource, trainingModel, testSource, testModel, numberOfKnownModels, numberOfCandidates); } else { correctModel = new OfflineTraining().getCorrectModel(contextParameters, trainingSources, trainingModels, testSource, testModel, numberOfCandidates); } } List<ColumnNode> columnNodes = correctModel.getColumnNodes(); // if (useCorrectType && numberOfCRFCandidates > 1) // updateCrfSemanticTypesForResearchEvaluation(columnNodes); List<Node> steinerNodes = new LinkedList<Node>(columnNodes); modelLearner = new ModelLearner_KnownModels(modelLearningGraph.getGraphBuilder(), steinerNodes); long start = System.currentTimeMillis(); String graphName = !iterativeEvaluation ? graphPath + semanticModels.get(newSourceIndex).getName() + Params.GRAPH_JSON_FILE_EXT : graphPath + semanticModels.get(newSourceIndex).getName() + ".knownModels=" + numberOfKnownModels + Params.GRAPH_JSON_FILE_EXT; if (new File(graphName).exists()) { // read graph from file try { logger.info("loading the graph ..."); DirectedWeightedMultigraph<Node, DefaultLink> graph = GraphUtil.importJson(graphName); GraphBuilder gb = new GraphBuilderTopK(ontologyManager, graph); modelLearner = new ModelLearner_KnownModels(gb, steinerNodes); } catch (Exception e) { e.printStackTrace(); } } else { logger.info("building the graph ..."); for (SemanticModel sm : trainingData) // modelLearningGraph.addModel(sm); modelLearningGraph.addModelAndUpdate(sm, PatternWeightSystem.JWSPaperFormula); modelLearner = new ModelLearner_KnownModels(modelLearningGraph.getGraphBuilder(), steinerNodes); // modelLearner.graphBuilder = modelLearningGraph.getGraphBuilder(); // modelLearner.nodeIdFactory = modelLearner.graphBuilder.getNodeIdFactory(); // save graph to file try { // GraphUtil.exportJson(modelLearningGraph.getGraphBuilder().getGraph(), graphName, true, true); // GraphVizUtil.exportJGraphToGraphviz(modelLearner.graphBuilder.getGraph(), // "test", // true, // GraphVizLabelType.LocalId, // GraphVizLabelType.LocalUri, // false, // true, // graphName + ".dot"); } catch (Exception e) { e.printStackTrace(); } } List<SortableSemanticModel> hypothesisList = modelLearner.hypothesize(useCorrectType, numberOfCandidates); long elapsedTimeMillis = System.currentTimeMillis() - start; float elapsedTimeSec = elapsedTimeMillis / 1000F; int cutoff = 20;//ModelingConfiguration.getMaxCandidateModels(); List<SortableSemanticModel> topHypotheses = null; if (hypothesisList != null) { topHypotheses = hypothesisList.size() > cutoff ? hypothesisList.subList(0, cutoff) : hypothesisList; } Map<String, SemanticModel> models = new TreeMap<String, SemanticModel>(); ModelEvaluation me; models.put("1-correct model", correctModel); if (topHypotheses != null) for (int k = 0; k < topHypotheses.size(); k++) { SortableSemanticModel m = topHypotheses.get(k); me = m.evaluate(correctModel, onlyEvaluateInternalLinks, false); String label = "candidate " + k + "\n" + // (m.getSteinerNodes() == null ? "" : m.getSteinerNodes().getScoreDetailsString()) + "link coherence:" + (m.getLinkCoherence() == null ? "" : m.getLinkCoherence().getCoherenceValue()) + "\n"; label += (m.getSteinerNodes() == null || m.getSteinerNodes().getCoherence() == null) ? "" : "node coherence:" + m.getSteinerNodes().getCoherence().getCoherenceValue() + "\n"; label += "confidence:" + m.getConfidenceScore() + "\n"; label += m.getSteinerNodes() == null ? "" : "mapping score:" + m.getSteinerNodes().getScore() + "\n"; label += "cost:" + roundDecimals(m.getCost(), 6) + "\n" + // "-distance:" + me.getDistance() + "-precision:" + me.getPrecision() + "-recall:" + me.getRecall(); models.put(label, m); if (k == 0) { // first rank model System.out.println("number of known models: " + numberOfKnownModels + ", precision: " + me.getPrecision() + ", recall: " + me.getRecall() + ", time: " + elapsedTimeSec + ", accuracy: " + correctModel.getAccuracy() + ", mrr: " + correctModel.getMrr()); logger.info("number of known models: " + numberOfKnownModels + ", precision: " + me.getPrecision() + ", recall: " + me.getRecall() + ", time: " + elapsedTimeSec + ", accuracy: " + correctModel.getAccuracy() + ", mrr: " + correctModel.getMrr()); // resultFile.println("number of known models \t precision \t recall"); // resultFile.println(numberOfKnownModels + "\t" + me.getPrecision() + "\t" + me.getRecall()); String s = me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec + "\t" + correctModel.getAccuracy() + "\t" + correctModel.getMrr(); if (iterativeEvaluation) { if (resultsArray[numberOfKnownModels + 2].length() > 0) resultsArray[numberOfKnownModels + 2].append(" \t "); resultsArray[numberOfKnownModels + 2].append(s); } else { // s = newSource.getName() + "\t" + // me.getPrecision() + "\t" + // me.getRecall() + "\t" + // elapsedTimeSec + "\t" + // correctModel.getAccuracy() + "\t" + // correctModel.getMrr(); s = me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec; resultFile.println(s); } // resultFile.println(me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec); } } String outputPath = Params.OUTPUT_DIR; String outName = !iterativeEvaluation ? outputPath + semanticModels.get(newSourceIndex).getName() + Params.GRAPHVIS_OUT_DETAILS_FILE_EXT : outputPath + semanticModels.get(newSourceIndex).getName() + ".knownModels=" + numberOfKnownModels + Params.GRAPHVIS_OUT_DETAILS_FILE_EXT; GraphVizUtil.exportSemanticModelsToGraphviz(models, newSource.getName(), outName, GraphVizLabelType.LocalId, GraphVizLabelType.LocalUri, true, true); numberOfKnownModels++; if (zeroKnownModel) break; } } if (iterativeEvaluation) { for (StringBuffer s : resultsArray) resultFileIterative.println(s.toString()); resultFileIterative.close(); } else { resultFile.close(); } } public static void runUkHackEvaluation() throws Exception { String karmaHomeDir = "/Users/mohsen/karma-uk-hack/"; ServletContextParameterMap contextParameters = ContextParametersRegistry.getInstance() .registerByKarmaHome(karmaHomeDir); contextParameters.setParameterValue(ContextParameter.USER_DIRECTORY_PATH, karmaHomeDir); contextParameters.setParameterValue(ContextParameter.USER_CONFIG_DIRECTORY, karmaHomeDir + "config"); List<String> trainingSources = new ArrayList<String>(); trainingSources.add("www.gunsinternational.com"); trainingSources.add("www.alaskaslist.com"); trainingSources.add("www.dallasguns.com"); trainingSources.add("www.elpasoguntrader.com"); trainingSources.add("www.floridagunclassifieds.com"); List<String> testSources = new ArrayList<String>(); testSources.add("www.armslist.com"); testSources.add("www.floridaguntrader.com"); testSources.add("www.hawaiiguntrader.com"); testSources.add("www.montanagunclassifieds.com"); testSources.add("www.msguntrader.com"); testSources.add("www.tennesseegunexchange.com"); testSources.add("www.kyclassifieds.com"); testSources.add("www.nextechclassifieds.com"); testSources.add("www.shooterswap.com"); testSources.add("www.theoutdoorstrader.com"); List<SemanticModel> trainingModels = new ArrayList<SemanticModel>(); List<SemanticModel> testModels = new ArrayList<SemanticModel>(); List<String> trainingFiles = new LinkedList<String>(); List<String> testFiles = new LinkedList<String>(); String ukHacKDirStr = karmaHomeDir + "git/data/weapons/"; File ukHackDir = new File(ukHacKDirStr); File[] ukHackWebsites = ukHackDir.listFiles(); if (ukHackWebsites != null) { for (File ukHackWebsite : ukHackWebsites) { if (!ukHackWebsite.isDirectory()) continue; File[] ukHackWebsiteFiles = ukHackWebsite.listFiles(); String sourceName = ukHackWebsite.getName(); if (!trainingSources.contains(sourceName) && !testSources.contains(sourceName)) continue; if (ukHackWebsiteFiles != null) { for (File ukHackWebsiteFile : ukHackWebsiteFiles) { if (ukHackWebsiteFile.getName().equalsIgnoreCase("model.json")) { SemanticModel sm = null; try { sm = SemanticModel.readJson(ukHackWebsiteFile.getAbsolutePath()); } catch (Exception e) { System.out.println("Error in reading " + ukHackWebsiteFile.getAbsolutePath()); break; } sm.setName(sourceName); if (trainingSources.contains(sourceName)) { trainingModels.add(sm); trainingFiles.add(ukHackWebsiteFile.getAbsolutePath()); } else if (testSources.contains(sourceName)) { testModels.add(sm); testFiles.add(ukHackWebsiteFile.getAbsolutePath()); } } } } } } OntologyManager ontologyManager = new OntologyManager(contextParameters.getId()); File ff = new File(karmaHomeDir + "preloaded-ontologies/"); File[] files = ff.listFiles(); for (File f : files) { if (f.getName().startsWith(".") || f.isDirectory()) { continue; //Ignore . files } ontologyManager.doImport(f, "UTF-8"); } ontologyManager.updateCache(); ModelLearningGraph modelLearningGraph = null; ModelLearner_KnownModels modelLearner; boolean onlyGenerateSemanticTypeStatistics = true; boolean useCorrectType = true; boolean onlyEvaluateInternalLinks = false || useCorrectType; int numberOfCandidates = 1; if (onlyGenerateSemanticTypeStatistics) { System.out.println("=============================================="); System.out.println("training"); System.out.println("=============================================="); getStatistics(trainingModels, trainingFiles, 4); System.out.println("=============================================="); System.out.println("test"); System.out.println("=============================================="); getStatistics(testModels, testFiles, 4); return; } String filePath = karmaHomeDir + "result/"; String filename = ""; filename += "results"; filename += useCorrectType ? "-correct" : "-k=" + numberOfCandidates; filename += onlyEvaluateInternalLinks ? "-internal" : "-all"; filename += ".csv"; PrintWriter resultFile = null; resultFile = new PrintWriter(new File(filePath + filename)); resultFile.println("source \t p \t r \t t \t a \t m \n"); for (int i = 0; i < testModels.size(); i++) { // for (int i = 0; i <= 1; i++) { // int i = 0; { int targetSourceIndex = i; SemanticModel targetSource = testModels.get(targetSourceIndex); logger.info("======================================================"); logger.info(targetSource.getName() + "(#attributes:" + targetSource.getColumnNodes().size() + ")"); System.out .println(targetSource.getName() + "(#attributes:" + targetSource.getColumnNodes().size() + ")"); logger.info("======================================================"); modelLearningGraph = (ModelLearningGraphCompact) ModelLearningGraph.getEmptyInstance(ontologyManager, ModelLearningGraphType.Compact); SemanticModel correctModel; correctModel = targetSource; // correctModel.setAccuracy(1.0); // correctModel.setMrr(1.0); List<ColumnNode> columnNodes = correctModel.getColumnNodes(); // if (useCorrectType && numberOfCRFCandidates > 1) // updateCrfSemanticTypesForResearchEvaluation(columnNodes); List<Node> steinerNodes = new LinkedList<Node>(columnNodes); modelLearner = new ModelLearner_KnownModels(modelLearningGraph.getGraphBuilder(), steinerNodes); long start = System.currentTimeMillis(); String graphName = karmaHomeDir + "alignment-graph/graph.json"; // if (new File(graphName).exists()) { // // read graph from file // try { // logger.info("loading the graph ..."); // DirectedWeightedMultigraph<Node, DefaultLink> graph = GraphUtil.importJson(graphName); // GraphBuilder gb = new GraphBuilderTopK(ontologyManager, graph); // modelLearner = new ModelLearner_KnownModels(gb, steinerNodes); // } catch (Exception e) { // e.printStackTrace(); // } // } else { logger.info("building the graph ..."); for (SemanticModel sm : trainingModels) // modelLearningGraph.addModel(sm); modelLearningGraph.addModelAndUpdate(sm, PatternWeightSystem.JWSPaperFormula); modelLearner = new ModelLearner_KnownModels(modelLearningGraph.getGraphBuilder(), steinerNodes); // modelLearner.graphBuilder = modelLearningGraph.getGraphBuilder(); // modelLearner.nodeIdFactory = modelLearner.graphBuilder.getNodeIdFactory(); // save graph to file try { GraphUtil.exportJson(modelLearningGraph.getGraphBuilder().getGraph(), graphName, true, true); GraphVizUtil.exportJGraphToGraphviz(modelLearner.graphBuilder.getGraph(), "test", true, GraphVizLabelType.LocalId, GraphVizLabelType.LocalUri, false, true, graphName + ".dot"); } catch (Exception e) { e.printStackTrace(); } } List<SortableSemanticModel> hypothesisList = modelLearner.hypothesize(useCorrectType, numberOfCandidates); long elapsedTimeMillis = System.currentTimeMillis() - start; float elapsedTimeSec = elapsedTimeMillis / 1000F; int cutoff = 3;//ModelingConfiguration.getMaxCandidateModels(); List<SortableSemanticModel> topHypotheses = null; if (hypothesisList != null) { topHypotheses = hypothesisList.size() > cutoff ? hypothesisList.subList(0, cutoff) : hypothesisList; } Map<String, SemanticModel> models = new TreeMap<String, SemanticModel>(); ModelEvaluation me; models.put("1-correct model", correctModel); if (topHypotheses != null) { for (int k = 0; k < topHypotheses.size(); k++) { SortableSemanticModel m = topHypotheses.get(k); me = m.evaluate(correctModel, onlyEvaluateInternalLinks, false); String label = "candidate " + k + "\n" + // (m.getSteinerNodes() == null ? "" : m.getSteinerNodes().getScoreDetailsString()) + // "link coherence:" + (m.getLinkCoherence() == null ? "" : m.getLinkCoherence().getCoherenceValue()) + "\n"; "link coherence:" + (m.getLinkCoherence() == null ? "" : m.getCoherenceString()) + "\n"; label += (m.getSteinerNodes() == null || m.getSteinerNodes().getCoherence() == null) ? "" : "node coherence:" + m.getSteinerNodes().getCoherence().getCoherenceValue() + "\n"; label += "confidence:" + m.getConfidenceScore() + "\n"; label += m.getSteinerNodes() == null ? "" : "mapping score:" + m.getSteinerNodes().getScore() + "\n"; label += "cost:" + roundDecimals(m.getCost(), 6) + "\n" + // "-distance:" + me.getDistance() + "-precision:" + me.getPrecision() + "-recall:" + me.getRecall(); models.put(label, m); if (k == 0) { // first rank model System.out.println("number of known models: " + trainingModels.size() + ", precision: " + me.getPrecision() + ", recall: " + me.getRecall() + ", time: " + elapsedTimeSec + ", accuracy: " + correctModel.getAccuracy() + ", mrr: " + correctModel.getMrr()); logger.info("number of known models: " + trainingModels.size() + ", precision: " + me.getPrecision() + ", recall: " + me.getRecall() + ", time: " + elapsedTimeSec + ", accuracy: " + correctModel.getAccuracy() + ", mrr: " + correctModel.getMrr()); // resultFile.println("number of known models \t precision \t recall"); // resultFile.println(numberOfKnownModels + "\t" + me.getPrecision() + "\t" + me.getRecall()); String s = me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec + "\t" + correctModel.getAccuracy() + "\t" + correctModel.getMrr(); // s = newSource.getName() + "\t" + // me.getPrecision() + "\t" + // me.getRecall() + "\t" + // elapsedTimeSec + "\t" + // correctModel.getAccuracy() + "\t" + // correctModel.getMrr(); s = me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec; resultFile.println(s); // resultFile.println(me.getPrecision() + "\t" + me.getRecall() + "\t" + elapsedTimeSec); } } } String outName = karmaHomeDir + "output/" + targetSource.getName() + ".out.dot"; GraphVizUtil.exportSemanticModelsToGraphviz(models, targetSource.getName(), outName, GraphVizLabelType.LocalId, GraphVizLabelType.LocalUri, true, true); } resultFile.close(); } public static void main(String[] args) throws Exception { boolean uk_hack = true; if (uk_hack) runUkHackEvaluation(); else runResearchEvaluation(); } }