List of usage examples for org.dom4j Node getStringValue
String getStringValue();
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportHandler.java
/** * Procedure to populate the RDF metadata and ingest the files *//*from w ww . j ava 2 s .c o m*/ public boolean execute() throws Exception { if (filesPaths != null) { File file = null; // List the source files for (int i = 0; i < filesPaths.length; i++) { file = new File(filesPaths[i]); if (file.exists()) { listFile(filesMap, file); } } } String message = ""; Document doc = null; DamsURI damsURI = null; String oid = null; int fLen = rdfFiles.length; String currFile = null; SAXReader saxReader = new SAXReader(); for (int i = 0; i < fLen && !interrupted; i++) { currFile = rdfFiles[i].getName(); setStatus( "Processing external import for file " + currFile + " (" + (i + 1) + " of " + fLen + ") ... "); try { doc = saxReader.read(rdfFiles[i]); List<Node> nodes = doc.selectNodes("//@rdf:about"); for (int j = 0; j < nodes.size(); j++) { Node nUri = nodes.get(j); String iUri = nUri.getStringValue(); Node parentNode = nUri.getParent(); String nName = parentNode.getName(); if (iUri.endsWith("/COL") || !(iUri.startsWith("http") && iUri.indexOf("/ark:/") > 0)) { // Assign ARK if (nName.endsWith("Object") || nName.endsWith("Component") || nName.endsWith("File") || (((Element) parentNode).isRootElement() || (parentNode.getParent().isRootElement() && parentNode.getParent().getName().equals("RDF")))) { String objId = iUri; if (nName.endsWith("Component") || nName.endsWith("File")) { damsURI = DamsURI.toParts(iUri, null); objId = damsURI.getObject(); } String srcObjKey = objId + "::" + rdfFiles[i].getAbsolutePath(); oid = idsMap.get(srcObjKey); // Assign new ARK if (oid == null) { oid = getNewId(); idsMap.put(srcObjKey, oid); } if (nName.endsWith("Object")) { objId = oid; objRecords.put(objId, currFile); } else if (nName.endsWith("Component") || nName.endsWith("File")) { damsURI.setObject(oid); // XXX // Assign cid and fid for Component and FIle if required objId = damsURI.toString(); } else objId = oid; nUri.setText(objId); updateReference(doc, iUri, objId); } else { String field = null; Node tNode = null; String xPath = null; Map<String, String> props = new TreeMap<String, String>(); String elemXPath = parentNode.getPath(); if (nName.endsWith("Collection") || nName.endsWith("CollectionPart")) { // Retrieve the Collection record field = "title_tesim"; xPath = "dams:title/mads:Title/mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { // Loop through to locate the rdfs:label if not selected by xPath. Node n = parentNode.selectSingleNode("dams:title"); for (Iterator<Element> it = ((Element) n).elementIterator(); it.hasNext();) { Element elem = it.next(); if (elem.getNamespacePrefix().equals("mads") && elem.getName().equals("Title")) tNode = elem.selectSingleNode("mads:authoritativeLabel"); } } } /* else if (nName.endsWith("Language") || nName.endsWith("Authority") || nName.endsWith("Subject") || nName.endsWith("Name") || nName.endsWith("Topic") || nName.endsWith("GenreForm") || nName.endsWith("Temporal") || nName.endsWith("Geographic")){ // Subject, Authority records use mads:authoritativeLabel field = "name_tesim"; xPath = "mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); } */else if (nName.endsWith(COPYRIGHT)) { // Copyright records use dams:copyrightStatus, plus other properties in the next step. field = "status_tesim"; xPath = "dams:copyrightStatus"; tNode = parentNode.selectSingleNode(xPath); props = copyrightProperties(parentNode); } else if (nName.endsWith(LICENSE)) { // License records use dams:LicenseNote, plus other properties in the next step. field = "note_tesim"; xPath = "dams:licenseNote"; tNode = parentNode.selectSingleNode(xPath); props = licenseProperties(parentNode); } else if (nName.endsWith(OTHERRIGHTS)) { // Copyright records use dams:copyrightStatus, plus other properties in the next step. field = "otherRightsBasis_tesim"; xPath = "dams:otherRightsBasis"; tNode = parentNode.selectSingleNode(xPath); props = licenseProperties(parentNode); } else if (nName.endsWith(RELATEDRESOURCE)) { // RelatedResource records use dams:description, plus other properties in the next step. field = "description_tesim"; xPath = "dams:description"; tNode = parentNode.selectSingleNode(xPath); props = relatedResourceProperties(parentNode); } else if (nName.endsWith(SOURCECAPTURE)) { // SourceCapture records use dams:sourceType, plus other properties in the next step. field = "sourceType_tesim"; xPath = "dams:sourceType"; tNode = parentNode.selectSingleNode(xPath); props = sourceCaptureProperties(parentNode); } else if (elemXPath.indexOf("mads", elemXPath.lastIndexOf('/') + 1) >= 0) { // MADSScheme and Language if (nName.endsWith(MADSSCHEME) || nName.equals(LANGUAGE)) { field = "code_tesim"; xPath = "mads:code"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { field = "name_tesim"; xPath = "rdfs:label"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { // Loop through to locate the rdfs:label if not selected by xPath. for (Iterator<Element> it = ((Element) parentNode).elementIterator(); it .hasNext();) { Element elem = it.next(); if (elem.getNamespacePrefix().equals("rdfs") && elem.getName().equals("label")) tNode = elem; } } } } else { // Subject, Authority records use mads:authoritativeLabel field = "name_tesim"; xPath = "mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { // Try to use the mads:code for mapping when mads:authoritativeLabel is not available field = "code_tesim"; xPath = "mads:code"; tNode = parentNode.selectSingleNode(xPath); } // Mapping for mads:isMemberOfMADSScheme String madsScheme = null; Node madsSchemeNode = parentNode.selectSingleNode("mads:isMemberOfMADSScheme"); if (madsSchemeNode != null) { Node msValueNode = madsSchemeNode.selectSingleNode("@rdf:resource"); if (msValueNode != null) { madsScheme = madsSchemeNode.getStringValue(); props.put("scheme_tesim", madsScheme); } else if ((madsSchemeNode = madsSchemeNode .selectSingleNode("mads:MADSScheme")) != null && madsSchemeNode.hasContent()) { if ((msValueNode = madsSchemeNode .selectSingleNode("mads:code")) != null) { madsScheme = msValueNode.getText(); props.put("scheme_code_tesim", madsScheme); } else if ((msValueNode = madsSchemeNode .selectSingleNode("rdfs:label")) != null) { madsScheme = msValueNode.getText(); props.put("scheme_name_tesim", madsScheme); } } else { props.put("scheme_tesim", ""); } } else { props.put("scheme_tesim", null); } } } else { // XXX Other Rights records like Statute, License, Other Rights etc. field = "value_tesim"; xPath = "rdf:value"; tNode = parentNode.selectSingleNode(xPath); field = "code_tesim"; if (tNode == null) { xPath = "dams:code"; tNode = parentNode.selectSingleNode(xPath); } } if (tNode == null) { throw new Exception("Element " + xPath + " is missing from the " + nName + " record " + iUri + " in file " + currFile + "."); } updateDocument(doc, parentNode, field, tNode.getText(), props); } } else if (nName.endsWith("Object")) { objRecords.put(iUri, currFile); } } String dams4Rdf = doc.asXML(); logData("dams4_" + rdfFiles[i].getName(), dams4Rdf); // Ingest the records String subjectId = null; DamsURI objURI = null; List<DamsURI> objURIs = null; RDFStore graph = null; rdfStore = new RDFStore(); Model rdf = rdfStore.loadRDFXML(dams4Rdf); initHandler(); Model iRdf = null; int jLen = items.size(); for (int j = 0; j < jLen && !interrupted; j++) { graph = new RDFStore(); recordsCount++; // Add subject subjectId = items.get(j); try { setStatus("Processing metadata for record " + subjectId + " (" + (j + 1) + " of " + jLen + ") in file " + currFile + " ... "); boolean succeeded = false; objURIs = objects.get(subjectId); for (int k = 0; k < objURIs.size(); k++) { objURI = objURIs.get(k); iRdf = rdfStore.querySubject(objURI.toString()); graph.merge(iRdf); } // Update object //log.info(j + " ingesting record " + subjectId + ":\n" + graph.export(RDFStore.RDFXML_ABBREV_FORMAT) + "\n\n"); succeeded = damsClient.updateObject(subjectId, graph.export(RDFStore.RDFXML_ABBREV_FORMAT), Constants.IMPORT_MODE_ADD); if (!succeeded) { if (metadataFailed.indexOf(currFile) < 0) failedCount++; metadataFailed.append(subjectId + " (" + currFile + "), \n"); message = "Metadata import for record " + subjectId + " failed (" + (j + 1) + " of " + jLen + ") in file " + currFile + "."; setStatus(message); logError(message + "\n Error RDF: \n" + graph.export(RDFStore.RDFXML_ABBREV_FORMAT)); } else { recordsIngested.add(subjectId); message = "Metadata import for record " + subjectId + " succeeded (" + (j + 1) + " of " + jLen + ") in file " + currFile + ". "; setStatus(message); logMessage(message); log.info(message); // Update SOLR fre records ingested. updateSOLR(subjectId); } } catch (Exception e) { e.printStackTrace(); if (metadataFailed.indexOf(currFile) < 0) failedCount++; metadataFailed.append(subjectId + " (" + currFile + "), \n"); message = "Metadata import failed: " + e.getMessage(); setStatus(message + " (" + (j + 1) + " of " + jLen + ") in file " + currFile + "."); logError(message); } try { Thread.sleep(10); } catch (InterruptedException e) { e.printStackTrace(); interrupted = true; failedCount++; metadataFailed.append(subjectId + " (" + currFile + ") \n"); message = "Metadata import interrupted for subject " + subjectId + ". \n Error: " + e.getMessage() + "\n"; setStatus("Canceled"); clearSession(); logError(message); } } // Ingest the source file if (importOption.equalsIgnoreCase("metadataAndFiles")) { uploadFiles(rdf, currFile); } } catch (Exception e) { e.printStackTrace(); failedCount++; message = "Import failed for " + currFile + ": " + e.getMessage(); setStatus(message + " (" + (i + 1) + " of " + fLen + ")."); logError(message); } finally { // Update SOLR for files uploaded int iLen = objWithFiles.size(); for (int j = 0; j < iLen && !interrupted; j++) { updateSOLR(objWithFiles.get(j)); } } setProgressPercentage(((i + 1) * 100) / fLen); try { Thread.sleep(10); } catch (InterruptedException e) { e.printStackTrace(); interrupted = true; failedCount++; message = "Import interrupted for oject in " + currFile + ". \n Error: " + e.getMessage() + "\n"; setStatus("Canceled"); clearSession(); logError(message); } } return exeResult; }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportHandler.java
/** * Update the document for DAMS ARK ID//from w w w . jav a 2 s.c om * @param doc * @param record * @param title * @throws Exception */ private void updateDocument(Document doc, Node record, String field, String title, Map<String, String> props) throws Exception { // Skip if the record detached if (record.getDocument() == null) return; // Subject, Authority records use mads:authoritativeLabel Node aboutAttr = record.selectSingleNode("@rdf:about"); String srcUri = aboutAttr.getStringValue(); //String nName = record.getName(); String xPath = record.getPath(); String elemName = xPath.substring(xPath.lastIndexOf("/") + 1); // MADSScheme model: MadsScheme if (elemName.endsWith("MADSScheme")) elemName = elemName.replace("MADSScheme", "Scheme"); String modelName = (elemName.substring(0, 1).toUpperCase() + elemName.substring(1)).replace(":", ""); String nKey = INFO_MODEL_PREFIX + modelName + "::" + title; if (props != null) { for (Iterator<String> it = props.keySet().iterator(); it.hasNext();) { String iKey = it.next(); nKey += "_" + iKey + "::" + props.get(iKey); } } String oid = idsMap.get(nKey); // Retrieve the record if (oid == null) { /*Map<String, String> props = null; if(nName.endsWith(COPYRIGHT)){ props = copyrightProperties(record); }*/ oid = lookupRecord(damsClient, field, title, modelName, props); if (oid == null) { // Create the record oid = getNewId(); aboutAttr.setText(oid); idsMap.put(nKey, oid); log.info("Found new redord " + srcUri + " (" + oid + ": " + field + " -- " + title); } else { // Record found. Add linking, remove it. toResourceLinking(oid, record); } } else { // Record added. Add linking, remove it. toResourceLinking(oid, record); } updateReference(doc, srcUri, oid); }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportHandler.java
public void updateReference(Document doc, String srcId, String oid) { List<Node> resNodes = doc.selectNodes("//@rdf:resource"); for (int k = 0; k < resNodes.size(); k++) { Node nRes = resNodes.get(k); if (nRes.getStringValue().equals(srcId)) { nRes.setText(oid);//w w w . j a v a2 s . c o m } } }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportTsHandler.java
/** * Procedure to populate the RDF metadata and ingest the files *///from w ww. java2 s . co m @Override public boolean execute() throws Exception { if (filesPaths != null) { File file = null; // List the source files for (int i = 0; i < filesPaths.length; i++) { file = new File(filesPaths[i]); if (file.exists()) { listFile(filesMap, file); } } } String message = ""; Document doc = null; DamsURI damsURI = null; List<String> recordsToReplace = null; String oid = null; int fLen = rdfFiles.length; String currFile = null; SAXReader saxReader = new SAXReader(); for (int i = 0; i < fLen && !interrupted; i++) { if (i == 0) logMessage("Object Import status:\n[Object title] - [URI] - [Status] - [Timestamp]"); recordsToReplace = new ArrayList<>(); currFile = rdfFiles[i].getName(); preprocessedTimestamp = damsDateFormat.format(rdfFiles[i].lastModified()); setStatus( "Processing external import for file " + currFile + " (" + (i + 1) + " of " + fLen + ") ... "); try { doc = saxReader.read(rdfFiles[i]); List<Node> nodes = doc.selectNodes("//@rdf:about"); for (int j = 0; j < nodes.size(); j++) { Node nUri = nodes.get(j); String iUri = nUri.getStringValue(); Node parentNode = nUri.getParent(); String nName = parentNode.getName(); if (iUri.endsWith("/COL") || !(iUri.startsWith("http") && iUri.indexOf("/ark:/") > 0)) { // Assign ARK if (nName.endsWith("Object") || nName.endsWith("Component") || nName.endsWith("File") || (((Element) parentNode).isRootElement() || (parentNode.getParent().isRootElement() && parentNode.getParent().getName().equals("RDF")))) { String objId = iUri; if (nName.endsWith("Component") || nName.endsWith("File")) { damsURI = DamsURI.toParts(iUri, null); objId = damsURI.getObject(); } String srcObjKey = objId + "::" + rdfFiles[i].getAbsolutePath(); oid = idsMap.get(srcObjKey); // Assign new ARK if (oid == null) { oid = getNewId(); idsMap.put(srcObjKey, oid); } if (nName.endsWith("Object")) { objId = oid; objRecords.put(objId, currFile); } else if (nName.endsWith("Component") || nName.endsWith("File")) { damsURI.setObject(oid); // XXX // Assign cid and fid for Component and FIle if required objId = damsURI.toString(); } else objId = oid; nUri.setText(objId); updateReference(doc, iUri, objId); } else { String field = null; Node tNode = null; String xPath = null; Map<String, String> props = new TreeMap<String, String>(); String elemXPath = parentNode.getPath(); if (nName.endsWith("Collection") || nName.endsWith("CollectionPart")) { // Retrieve the Collection record field = "dams:title/mads:authoritativeLabel"; xPath = "dams:title/mads:Title/mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { // Loop through to locate the rdfs:label if not selected by xPath. Node n = parentNode.selectSingleNode("dams:title"); for (Iterator<Element> it = ((Element) n).elementIterator(); it.hasNext();) { Element elem = it.next(); if (elem.getNamespacePrefix().equals("mads") && elem.getName().equals("Title")) tNode = elem.selectSingleNode("mads:authoritativeLabel"); } } } /* else if (nName.endsWith("Language") || nName.endsWith("Authority") || nName.endsWith("Subject") || nName.endsWith("Name") || nName.endsWith("Topic") || nName.endsWith("GenreForm") || nName.endsWith("Temporal") || nName.endsWith("Geographic")){ // Subject, Authority records use mads:authoritativeLabel field = "name_tesim"; xPath = "mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); } */else if (nName.endsWith(COPYRIGHT)) { // Copyright records use dams:copyrightStatus, plus other properties in the next step. field = "dams:copyrightStatus"; xPath = "dams:copyrightStatus"; tNode = parentNode.selectSingleNode(xPath); props = copyrightProperties(parentNode); } else if (nName.endsWith(LICENSE)) { // License records use dams:LicenseNote, plus other properties in the next step. field = "dams:licenseNote"; xPath = "dams:licenseNote"; tNode = parentNode.selectSingleNode(xPath); props = licenseProperties(parentNode); } else if (nName.endsWith(OTHERRIGHTS)) { // Copyright records use dams:copyrightStatus, plus other properties in the next step. field = "dams:otherRightsBasis"; xPath = "dams:otherRightsBasis"; tNode = parentNode.selectSingleNode(xPath); props = otherRightsProperties(parentNode); } else if (nName.endsWith(RELATEDRESOURCE)) { // RelatedResource records use dams:description, plus other properties in the next step. field = "dams:description"; xPath = "dams:description"; tNode = parentNode.selectSingleNode(xPath); props = relatedResourceProperties(parentNode); } else if (nName.endsWith(SOURCECAPTURE)) { // SourceCapture records use dams:sourceType, plus other properties in the next step. field = "dams:sourceType"; xPath = "dams:sourceType"; tNode = parentNode.selectSingleNode(xPath); props = sourceCaptureProperties(parentNode); } else if (nName.endsWith(NOTE)) { // Note records use rdf:value, dams:type, dams:displayLabel. field = "rdf:value"; xPath = "rdf:value"; tNode = parentNode.selectSingleNode(xPath); props = noteProperties(parentNode); } else if (nName.endsWith(PERMISSION) || nName.equals(RESTRICTION)) { field = "dams:type"; xPath = "dams:type"; tNode = parentNode.selectSingleNode(xPath); props = dateProperties(parentNode); } else if (elemXPath.indexOf("mads", elemXPath.lastIndexOf('/') + 1) >= 0) { // MADSScheme and Language if (nName.endsWith(MADSSCHEME)) { field = "mads:code"; xPath = "mads:code"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { field = "rdfs:label"; xPath = "rdfs:label"; tNode = parentNode.selectSingleNode("*[name()='" + xPath + "']"); } } else if (nName.endsWith(LANGUAGE)) { field = "mads:code"; xPath = "mads:code"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { field = "mads:authoritativeLabel"; xPath = "mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); } } else { // Subject, Authority records use mads:authoritativeLabel field = "mads:authoritativeLabel"; xPath = "mads:authoritativeLabel"; tNode = parentNode.selectSingleNode(xPath); if (tNode == null) { // Try to use the mads:code for mapping when mads:authoritativeLabel is not available field = "mads:code"; xPath = "mads:code"; tNode = parentNode.selectSingleNode(xPath); } else { Node diplayLabelNode = parentNode .selectSingleNode("*[name()='dams:displayLabel']"); props.put("dams:displayLabel", encodeLiteralValue(diplayLabelNode)); } // Mapping for mads:isMemberOfMADSScheme String madsScheme = null; Node madsSchemeNode = parentNode.selectSingleNode("mads:isMemberOfMADSScheme"); if (madsSchemeNode != null) { Node msValueNode = madsSchemeNode.selectSingleNode("@rdf:resource"); if (msValueNode != null) { madsScheme = madsSchemeNode.getStringValue(); props.put("mads:isMemberOfMADSScheme", "<" + madsScheme + ">"); } else if ((madsSchemeNode = madsSchemeNode .selectSingleNode("mads:MADSScheme")) != null && madsSchemeNode.hasContent()) { if ((msValueNode = madsSchemeNode .selectSingleNode("mads:code")) != null) { madsScheme = encodeLiteralValue(msValueNode); props.put("mads:isMemberOfMADSScheme/mads:code", madsScheme); } else if ((msValueNode = madsSchemeNode .selectSingleNode("rdfs:label")) != null) { madsScheme = encodeLiteralValue(msValueNode); props.put("mads:isMemberOfMADSScheme/rdfs:label", madsScheme); } } else { props.put("mads:isMemberOfMADSScheme/rdfs:label", "\"\""); } } else { props.put("mads:isMemberOfMADSScheme/rdfs:label", null); } } } else { // XXX Other Rights records like Statute, License, Other Rights etc. field = "rdf:value"; xPath = "rdf:value"; tNode = parentNode.selectSingleNode(xPath); field = "dams:code"; if (tNode == null) { xPath = "dams:code"; tNode = parentNode.selectSingleNode(xPath); } } if (tNode == null && !field.equals("dams:licenseNote")) { throw new Exception("Element " + xPath + " is missing from the " + nName + " record " + iUri + " in file " + currFile + "."); } updateDocument(doc, parentNode, field, encodeLiteralValue(tNode), props); } } else { if (nName.endsWith("Object")) objRecords.put(iUri, currFile); if (replace && !(nName.endsWith("Component") || nName.endsWith("File"))) recordsToReplace.add(iUri); } } String dams4Rdf = doc.asXML(); logData("dams4_" + rdfFiles[i].getName(), dams4Rdf); // Ingest the records String subjectId = null; DamsURI objURI = null; List<DamsURI> objURIs = null; RDFStore graph = null; rdfStore = new RDFStore(); rdfStore.loadRDFXML(dams4Rdf); initHandler(); Model iRdf = null; items = sortRecords(items); int jLen = items.size(); //System.out.println(currFile + " records found: " + jLen); for (int j = 0; j < jLen && !interrupted; j++) { processIndex = 0; status = new boolean[processNames.length]; messages = new StringBuilder[processNames.length]; for (int k = 0; k < messages.length; k++) { messages[k] = new StringBuilder(); } Model objModel = null; graph = new RDFStore(); recordsCount++; // Add subject subjectId = items.get(j); try { setStatus("Processing metadata for record " + subjectId + " (" + (j + 1) + " of " + jLen + ") in file " + currFile + " ... "); boolean succeeded = false; objURIs = objects.get(subjectId); for (int k = 0; k < objURIs.size(); k++) { objURI = objURIs.get(k); iRdf = rdfStore.querySubject(objURI.toString()); objModel = graph.merge(iRdf); } // Update object //log.info(j + " ingesting record " + subjectId + ":\n" + graph.export(RDFStore.RDFXML_ABBREV_FORMAT) + "\n\n"); String importMode = Constants.IMPORT_MODE_ADD; if (replace && recordsToReplace.indexOf(subjectId) >= 0) importMode = Constants.IMPORT_MODE_ALL; succeeded = damsClient.updateObject(subjectId, graph.export(RDFStore.RDFXML_ABBREV_FORMAT), importMode); // Logging for Object RDF/XML validation status[processIndex] = succeeded; messages[processIndex].append(damsDateFormat.format(new Date())); if (!succeeded) { if (metadataFailed.indexOf(currFile) < 0) failedCount++; metadataFailed.append(subjectId + " (" + currFile + "), \n"); message = "Metadata import for record " + subjectId + " failed (" + (j + 1) + " of " + jLen + ") in file " + currFile + "."; setStatus(message); log.error(message + "\n Error RDF: \n" + graph.export(RDFStore.RDFXML_ABBREV_FORMAT)); } else { recordsIngested.add(subjectId); message = "Metadata import for record " + subjectId + " succeeded (" + (j + 1) + " of " + jLen + ") in file " + currFile + ". "; setStatus(message); log.info(message); processIndex++; status[processIndex] = succeeded; messages[processIndex].append(damsDateFormat.format(new Date())); // Ingest the source file only if metadata ingested successfully if (status[processIndex] && importOption.equalsIgnoreCase("metadataAndFiles")) { uploadFiles(objModel, currFile, subjectId); } } } catch (Exception e) { e.printStackTrace(); if (metadataFailed.indexOf(currFile) < 0) failedCount++; metadataFailed.append(subjectId + " (" + currFile + "), \n"); message = "Metadata import failed: " + e.getMessage(); setStatus(message + " (" + (j + 1) + " of " + jLen + ") in file " + currFile + "."); log.error(message); String error = e.getMessage(); if (error.indexOf("Invalid RDF input") >= 0) { messages[processIndex].append(error); } else { status[processIndex] = true; messages[processIndex].append(damsDateFormat.format(new Date())); processIndex++; messages[processIndex].append(error); } } finally { int solrRequestIndex = processNames.length - 1; try { // Update SOLR for the record. status[solrRequestIndex] = updateSOLR(subjectId); messages[solrRequestIndex].append(damsDateFormat.format(new Date())); log.info("SOLR update requested for " + subjectId + ": " + damsClient.getRequestURL() + " " + status[solrRequestIndex]); } catch (Exception e) { e.printStackTrace(); exeResult = false; log.error("SOLR Index failed " + subjectId + ": " + e.getMessage()); messages[processNames.length - 1].append(e.getMessage()); } if (exeResult) exeResult = status[processIndex]; String resultMessage = "http://" + Constants.CLUSTER_HOST_NAME + ".ucsd.edu/dc/object/" + subjectId.substring(subjectId.lastIndexOf("/") + 1) + " - " + (status[processIndex] && status[solrRequestIndex] ? "successful" : "failed") + " - " + damsDateFormat.format(new Date()); if (objRecords.containsKey(subjectId)) { String title = getTitle(objModel, subjectId); if (StringUtils.isBlank(title)) title = "[Unknown Title]"; logMessage("\n" + title + " - " + resultMessage); if (!status[processIndex] || !status[solrRequestIndex]) { // Logging for pre-procesing - succeeded. logMessage("* Pre-processing - successful - " + preprocessedTimestamp); for (int k = 0; k <= processIndex; k++) { if (status[k] || !status[k] && status[k - 1]) { logMessage("* " + processNames[k] + " - " + (status[k] ? "successful" : "failed") + " - " + messages[k].toString()); } } // SOLR index request logging if (!status[solrRequestIndex]) logMessage("* " + processNames[solrRequestIndex] + " - " + (status[solrRequestIndex] ? "successful" : "failed") + " - " + messages[solrRequestIndex].toString()); } } else { ingestMessages.append("\n" + resultMessage); if (!status[processIndex]) { for (int k = 0; k + 1 < processIndex; k++) { if (status[k] || !status[k] && status[k - 1]) { logMessage("* " + processNames[k] + " - " + (status[k] ? "successful" : "failed") + " - " + messages[k].toString()); } } } } } try { Thread.sleep(10); } catch (InterruptedException e) { e.printStackTrace(); interrupted = true; exeResult = false; failedCount++; metadataFailed.append(subjectId + " (" + currFile + ") \n"); message = "Metadata import interrupted for subject " + subjectId + ". \n Error: " + e.getMessage() + "\n"; setStatus("Canceled"); clearSession(); log.error(message); logMessage("Client Cancled - " + damsDateFormat.format(new Date())); } } } catch (Exception e) { e.printStackTrace(); exeResult = false; failedCount++; message = "Import failed for " + currFile + ": " + e.getMessage(); setStatus(message + " (" + (i + 1) + " of " + fLen + ")."); log.error(message); } setProgressPercentage(((i + 1) * 100) / fLen); try { Thread.sleep(10); } catch (InterruptedException e) { e.printStackTrace(); exeResult = false; interrupted = true; failedCount++; message = "Import interrupted for oject in " + currFile + ". \n Error: " + e.getMessage() + "\n"; setStatus("Canceled"); clearSession(); log.error(message); messages[processIndex].append("Client canceled - " + damsDateFormat.format(new Date())); } } return exeResult; }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportTsHandler.java
/** * Update the document for DAMS ARK ID//from w w w .j a v a 2s .co m * @param doc * @param record * @param title * @throws Exception */ private void updateDocument(Document doc, Node record, String field, String title, Map<String, String> props) throws Exception { // Skip if the record detached if (record.getDocument() == null) return; // Subject, Authority records use mads:authoritativeLabel Node aboutAttr = record.selectSingleNode("@rdf:about"); String srcUri = aboutAttr.getStringValue(); //String nName = record.getName(); String xPath = record.getPath(); String elemName = xPath.substring(xPath.lastIndexOf("/") + 1); String modelName = elemName; String nKey = INFO_MODEL_PREFIX + modelName + "::" + title; if (props != null) { for (Iterator<String> it = props.keySet().iterator(); it.hasNext();) { String iKey = it.next(); nKey += "_" + iKey + "::" + props.get(iKey); } } String oid = idsMap.get(nKey); if (oid == null) { //Lookup records from the triplestore, matching the required properties that are null or empty. List<Map<String, String>> oids = lookupRecordsFromTs(field, title, "\"" + modelName + "\"", props); if (oids != null && oids.size() > 0) { String propName = null; String propValue = null; Document recDoc = null; Node cNode = null; if (props != null) { List<Map<String, String>> oidsCopy = new ArrayList<Map<String, String>>(); oidsCopy.addAll(oids); for (int i = 0; i < oidsCopy.size(); i++) { Collection<String> propValues = props.values(); Map<String, String> resolution = oidsCopy.get(i); String rid = resolution.values().iterator().next(); if (rid.startsWith("http")) { if (propValues.contains(null)) { recDoc = damsClient.getRecord(rid); for (Iterator<String> it = props.keySet().iterator(); it.hasNext();) { propName = it.next(); propValue = props.get(propName); // Test for the nodes for null properties and remove it from the result if (propValue == null) { int idx = propName.indexOf("/", 1); if (idx > 0) cNode = recDoc.selectSingleNode("//" + modelName + "/" + propName); else cNode = recDoc.selectSingleNode("//" + modelName + "/" + propName); if (cNode != null) { oids.remove(resolution); break; } } } } } else // removed internal BlankNodes from the results oids.remove(resolution); } } if (oids.size() > 0) { oid = oids.get(0).values().iterator().next(); if (oids.size() > 1) { String duids = ""; for (Iterator<Map<String, String>> it = oids.iterator(); it.hasNext();) duids += (duids.length() > 0 ? ", " : "") + it.next().values().iterator().next(); log.warn("Duplicated records found for " + title + " (" + field + "): " + duids + "."); } } } if (oid == null) { // Create the record oid = getNewId(); aboutAttr.setText(oid); } else { // Record found. Add to the map, link and remove it. toResourceLinking(oid, record); } idsMap.put(nKey, oid); } else { // Record added. Add linking, remove it. toResourceLinking(oid, record); } updateReference(doc, srcUri, oid); }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportTsHandler.java
private Map<String, String> getProperties(Node record, List<String> propNames) { Map<String, String> props = new TreeMap<String, String>(); String propName = null;/*from w w w . ja v a 2s .c om*/ String propValue = null; for (Iterator<String> it = propNames.iterator(); it.hasNext();) { propValue = null; propName = it.next(); Node tNode = record.selectSingleNode(propName); if (tNode != null) { Node resNode = tNode.selectSingleNode("@rdf:resource"); if (resNode != null) propValue = encodeIdentifier(resNode.getStringValue()); else propValue = encodeLiteralValue(tNode); } props.put(propName, propValue); } return props; }
From source file:edu.ucsd.library.xdre.imports.RDFDAMS4ImportTsHandler.java
private String encodeLiteralValue(Node node) { String value = null;/*w ww . ja v a 2 s .c o m*/ if (node != null) { value = "\"" + StringEscapeUtils.escapeJava(Normalizer.normalize(node.getText(), Normalizer.Form.NFC)) .replace("'", "\\'").replace("\"", "\\\"") + "\""; Node langAttr = node.selectSingleNode("@xml:lang"); if (langAttr != null) value += "@" + langAttr.getStringValue(); } return value; }
From source file:fr.cph.stock.language.Language.java
License:Apache License
/** * Get the language map/*w w w. j a v a2s . c o m*/ * * @return a map */ public final Map<String, String> getLanguage() { Node menuNode = xml.getNode(BASE + MENU); @SuppressWarnings("unchecked") List<Node> menuNodes = (List<Node>) xml.getListNode(BASE + MENU); for (Node node : menuNodes) { String name = node.valueOf("@name"); map.put(menuNode.getName().toUpperCase() + "_" + name.toUpperCase(), node.getStringValue()); } Node node = xml.getNode(BASE + PORTFOLIO); List<String> nodes = new ArrayList<>(); Node node2; nodes.add(BASE + PORTFOLIO + "/title"); nodes.add(BASE + PORTFOLIO + "/review/liquidity"); nodes.add(BASE + PORTFOLIO + "/review/yieldYear"); nodes.add(BASE + PORTFOLIO + "/review/shareValue"); nodes.add(BASE + PORTFOLIO + "/add"); nodes.add(BASE + PORTFOLIO + "/equities/company"); nodes.add(BASE + PORTFOLIO + "/equities/quantity"); nodes.add(BASE + PORTFOLIO + "/equities/unitCostPrice"); nodes.add(BASE + PORTFOLIO + "/equities/quote"); nodes.add(BASE + PORTFOLIO + "/equities/currency"); nodes.add(BASE + PORTFOLIO + "/equities/parities"); nodes.add(BASE + PORTFOLIO + "/equities/value"); nodes.add(BASE + PORTFOLIO + "/equities/percentTotal"); nodes.add(BASE + PORTFOLIO + "/equities/yieldTtm"); nodes.add(BASE + PORTFOLIO + "/equities/yieldPerUnitCostPrice"); nodes.add(BASE + PORTFOLIO + "/equities/valueGained"); nodes.add(BASE + PORTFOLIO + "/equities/stopLoss"); nodes.add(BASE + PORTFOLIO + "/equities/objective"); nodes.add(BASE + PORTFOLIO + "/equities/info"); nodes.add(BASE + PORTFOLIO + "/equities/info"); nodes.add(BASE + PORTFOLIO + "/equities/modify"); nodes.add(BASE + PORTFOLIO + "/chartTitle"); nodes.add(BASE + PORTFOLIO + "/chartTitleValue"); nodes.add(BASE + PORTFOLIO + "/chart/all"); nodes.add(BASE + PORTFOLIO + "/chart/fiveYears"); nodes.add(BASE + PORTFOLIO + "/chart/twoYears"); nodes.add(BASE + PORTFOLIO + "/chart/oneYear"); nodes.add(BASE + PORTFOLIO + "/chart/sixMonths"); nodes.add(BASE + PORTFOLIO + "/chart/threeMonths"); nodes.add(BASE + PORTFOLIO + "/chart/oneMonth"); nodes.add(BASE + PORTFOLIO + "/chart/oneWeek"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + CONSTANT); nodes = new ArrayList<>(); nodes.add(BASE + CONSTANT + "/added"); nodes.add(BASE + CONSTANT + "/updated"); nodes.add(BASE + CONSTANT + "/modified"); nodes.add(BASE + CONSTANT + "/deleted"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + PORTFOLIO_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + PORTFOLIO_HIDDEN + "/refresh"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/add"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/updateCurrencies"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/yahooId"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/quantity"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/unitCostPrice"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/modifyEquity"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/customizeName"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/customizeName"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/customizeSector"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/customizeIndustry"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/customizeMarketCap"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/defautParity"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/personalParity"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/yahooYield"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/personalYield"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/stopLoss"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/objective"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/modify"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/or"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/delete"); nodes.add(BASE + PORTFOLIO_HIDDEN + "/deleteConfirm"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + HISTORY); nodes = new ArrayList<>(); nodes.add(BASE + HISTORY + "/title"); nodes.add(BASE + HISTORY + "/update"); nodes.add(BASE + HISTORY + "/date"); nodes.add(BASE + HISTORY + "/account"); nodes.add(BASE + HISTORY + "/liquidityMovement"); nodes.add(BASE + HISTORY + "/dividends"); nodes.add(BASE + HISTORY + "/buy"); nodes.add(BASE + HISTORY + "/sell"); nodes.add(BASE + HISTORY + "/taxe"); nodes.add(BASE + HISTORY + "/portfolioValue"); nodes.add(BASE + HISTORY + "/shareQuantity"); nodes.add(BASE + HISTORY + "/shareValue"); nodes.add(BASE + HISTORY + "/monthlyYield"); nodes.add(BASE + HISTORY + "/commentary"); nodes.add(BASE + HISTORY + "/option"); nodes.add(BASE + HISTORY + "/details"); nodes.add(BASE + HISTORY + "/all"); nodes.add(BASE + HISTORY + "/delete"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + HISTORY_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + HISTORY_HIDDEN + "/refresh"); nodes.add(BASE + HISTORY_HIDDEN + "/account"); nodes.add(BASE + HISTORY_HIDDEN + "/movement"); nodes.add(BASE + HISTORY_HIDDEN + "/yield"); nodes.add(BASE + HISTORY_HIDDEN + "/buy"); nodes.add(BASE + HISTORY_HIDDEN + "/sell"); nodes.add(BASE + HISTORY_HIDDEN + "/taxe"); nodes.add(BASE + HISTORY_HIDDEN + "/commentary"); nodes.add(BASE + HISTORY_HIDDEN + "/update"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + ACCOUNTS); nodes = new ArrayList<>(); nodes.add(BASE + ACCOUNTS + "/title"); nodes.add(BASE + ACCOUNTS + "/add"); nodes.add(BASE + ACCOUNTS + "/account"); nodes.add(BASE + ACCOUNTS + "/currency"); nodes.add(BASE + ACCOUNTS + "/liquidity"); nodes.add(BASE + ACCOUNTS + "/parity"); nodes.add(BASE + ACCOUNTS + "/value"); nodes.add(BASE + ACCOUNTS + "/option"); nodes.add(BASE + ACCOUNTS + "/modify"); nodes.add(BASE + ACCOUNTS + "/total"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + ACCOUNTS_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + ACCOUNTS_HIDDEN + "/add"); nodes.add(BASE + ACCOUNTS_HIDDEN + "/accountName"); nodes.add(BASE + ACCOUNTS_HIDDEN + "/currency"); nodes.add(BASE + ACCOUNTS_HIDDEN + "/liquidity"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + LIST); nodes = new ArrayList<>(); nodes.add(BASE + LIST + "/title"); nodes.add(BASE + LIST + "/add"); nodes.add(BASE + LIST + "/date"); nodes.add(BASE + LIST + "/company"); nodes.add(BASE + LIST + "/quote"); nodes.add(BASE + LIST + "/lowMaxYear"); nodes.add(BASE + LIST + "/yield"); nodes.add(BASE + LIST + "/lowerLimit"); nodes.add(BASE + LIST + "/higherLimit"); nodes.add(BASE + LIST + "/info"); nodes.add(BASE + LIST + "/modify"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + LIST_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + LIST_HIDDEN + "/refresh"); nodes.add(BASE + LIST_HIDDEN + "/confirm"); nodes.add(BASE + LIST_HIDDEN + "/add"); nodes.add(BASE + LIST_HIDDEN + "/yahooId"); nodes.add(BASE + LIST_HIDDEN + "/lowerLimit"); nodes.add(BASE + LIST_HIDDEN + "/higherLimit"); nodes.add(BASE + LIST_HIDDEN + "/modify"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + CHARTS); nodes = new ArrayList<>(); nodes.add(BASE + CHARTS + "/title"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + CURRENCIES); nodes = new ArrayList<>(); nodes.add(BASE + CURRENCIES + "/title"); nodes.add(BASE + CURRENCIES + "/portfolioCurrency"); nodes.add(BASE + CURRENCIES + "/currency"); nodes.add(BASE + CURRENCIES + "/lastUpdate"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + CURRENCIES_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + CURRENCIES_HIDDEN + "/refresh"); nodes.add(BASE + CURRENCIES_HIDDEN + "/confirm"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + OPTIONS); nodes = new ArrayList<>(); nodes.add(BASE + OPTIONS + "/title"); nodes.add(BASE + OPTIONS + "/defautCurrency"); nodes.add(BASE + OPTIONS + "/format"); nodes.add(BASE + OPTIONS + "/timeZone"); nodes.add(BASE + OPTIONS + "/datePattern"); nodes.add(BASE + OPTIONS + "/portfolioColumn"); nodes.add(BASE + OPTIONS + "/loadHistory"); nodes.add(BASE + OPTIONS + "/liquidity"); nodes.add(BASE + OPTIONS + "/account"); nodes.add(BASE + OPTIONS + "/load"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } node = xml.getNode(BASE + OPTIONS_HIDDEN); nodes = new ArrayList<>(); nodes.add(BASE + OPTIONS_HIDDEN + "/currency"); nodes.add(BASE + OPTIONS_HIDDEN + "/format"); nodes.add(BASE + OPTIONS_HIDDEN + "/timeZone"); nodes.add(BASE + OPTIONS_HIDDEN + "/datePattern"); nodes.add(BASE + OPTIONS_HIDDEN + "/columns"); nodes.add(BASE + OPTIONS_HIDDEN + "/quote"); nodes.add(BASE + OPTIONS_HIDDEN + "/parities"); nodes.add(BASE + OPTIONS_HIDDEN + "/yieldTTM"); nodes.add(BASE + OPTIONS_HIDDEN + "/yieldPerUnitCostPrice"); nodes.add(BASE + OPTIONS_HIDDEN + "/stopLoss"); nodes.add(BASE + OPTIONS_HIDDEN + "/objective"); for (String n : nodes) { node2 = xml.getNode(n); addToMap(map, node, node2); } return map; }
From source file:fr.cph.stock.language.Language.java
License:Apache License
/** * Add the informations to the result map * * @param m the result map//from ww w .java2s.co m * @param base the node base * @param node the node */ protected final void addToMap(final Map<String, String> m, final Node base, final Node node) { m.put(base.getName().toUpperCase() + "_" + node.getName().toUpperCase(), node.getStringValue()); }
From source file:gg.imports.GrisbiFile050.java
License:Open Source License
/** * Gets expected number of payees/*from w ww.j ava 2 s . co m*/ * @return Expected number of payees * @throws ParsingException Expected number of payees not found * @throws NumberFormatException The expected number of payees is not numeric */ private int getExpectedNumberOfPayees() throws ParsingException, NumberFormatException { log.entering(this.getClass().getName(), "getExpectedNumberOfPayees"); Node expectedNumberOfPayeesNode = grisbiFileDocument.selectSingleNode("/Grisbi/Tiers/Generalites/Nb_tiers"); if (expectedNumberOfPayeesNode == null) { throw new ParsingException("The expected number of payees has not been found in the Grisbi file '" + pathToGrisbiFile + "'"); } String expectedNumberOfPayeesValue = expectedNumberOfPayeesNode.getStringValue(); assert (expectedNumberOfPayeesValue != null); int expectedNumberOfPayees = Integer.parseInt(expectedNumberOfPayeesValue); log.exiting(this.getClass().getName(), "getExpectedNumberOfPayees", expectedNumberOfPayees); return expectedNumberOfPayees; }