List of usage examples for java.util LinkedHashSet remove
boolean remove(Object o);
From source file:org.chromium.content_shell.Shell.java
private void updateHistory(String url) { String json = mPref.getString("history", null); JSONArray array = new JSONArray(); if (json != null) { try {/*from w w w . j a va 2 s.co m*/ array = new JSONArray(json); } catch (JSONException e) { // TODO Auto-generated catch block e.printStackTrace(); } } LinkedHashSet<String> history = new LinkedHashSet<String>(); for (int i = 0; i < array.length(); i++) { try { history.add(array.getString(i)); } catch (JSONException e) { // TODO Auto-generated catch block e.printStackTrace(); } } if (history.contains(url)) { history.remove(url); } history.add(url); if (history.size() > 100) { String f = history.iterator().next(); history.remove(f); } array = new JSONArray(); for (String u : history) { array.put(u); } mPref.edit().putString("history", array.toString()).commit(); }
From source file:net.sf.taverna.t2.activities.beanshell.BeanshellActivityHealthChecker.java
public VisitReport visit(BeanshellActivity activity, List<Object> ancestors) { Object subject = (Processor) VisitReport.findAncestor(ancestors, Processor.class); if (subject == null) { // Fall back to using the activity itself as the subject of the reports subject = activity;//w w w . ja va2 s . c o m } List<VisitReport> reports = new ArrayList<VisitReport>(); String script = activity.getConfiguration().get("script").textValue(); if (!script.trim().endsWith(";")) { /** Missing ; on last line is not allowed by Parser, * but is allowed by Interpreter.eval() used at runtime */ script = script + ";"; } Parser parser = new Parser(new StringReader(script)); try { while (!parser.Line()) ; reports.add(new VisitReport(HealthCheck.getInstance(), subject, "Script OK", HealthCheck.NO_PROBLEM, Status.OK)); } catch (ParseException e) { VisitReport report = new VisitReport(HealthCheck.getInstance(), subject, e.getMessage(), HealthCheck.INVALID_SCRIPT, Status.SEVERE); report.setProperty("exception", e); reports.add(report); } // Check if we can find all the Beanshell's dependencies if (activity.getConfiguration().has("localDependency")) { LinkedHashSet<String> localDependencies = new LinkedHashSet<>(); for (JsonNode localDependency : activity.getConfiguration().get("localDependency")) { localDependencies.add(localDependency.textValue()); } String[] jarArray = activity.libDir.list(new FileExtFilter(".jar")); if (jarArray != null) { List<String> jarFiles = Arrays.asList(jarArray); // URLs of all jars found in the lib directory for (String jar : localDependencies) { if (jarFiles.contains(jar)) { localDependencies.remove(jar); } } } if (localDependencies.isEmpty()) { // all dependencies found reports.add(new VisitReport(HealthCheck.getInstance(), subject, "Beanshell dependencies found", HealthCheck.NO_PROBLEM, Status.OK)); } else { VisitReport vr = new VisitReport(HealthCheck.getInstance(), subject, "Beanshell dependencies missing", HealthCheck.MISSING_DEPENDENCY, Status.SEVERE); vr.setProperty("dependencies", localDependencies); vr.setProperty("directory", activity.libDir); reports.add(vr); } } Status status = VisitReport.getWorstStatus(reports); VisitReport report = new VisitReport(HealthCheck.getInstance(), subject, "Beanshell report", HealthCheck.NO_PROBLEM, status, reports); return report; }
From source file:net.sf.taverna.t2.activities.apiconsumer.ApiConsumerActivityHealthChecker.java
public VisitReport visit(ApiConsumerActivity subject, List<Object> ancestors) { // Check if we can find the jar containing the apiconsumer's class Processor p = (Processor) VisitReport.findAncestor(ancestors, Processor.class); if (p == null) { return null; }//from w w w .j av a2 s.c o m List<VisitReport> reports = new ArrayList<VisitReport>(); JsonNode configuration = subject.getConfiguration(); /* String className = configuration.getClassName(); try { // Try to load the API consumer's class ClassLoader classLoader = subject.getClassLoader(); classLoader.loadClass(className); reports.add(new VisitReport(HealthCheck.getInstance(), p, "Class found", HealthCheck.NO_PROBLEM, Status.OK)); // All is fine } catch (ClassNotFoundException e) { VisitReport vr = new VisitReport(HealthCheck.getInstance(), p, "Class missing", HealthCheck.MISSING_CLASS, Status.SEVERE); vr.setProperty("className", className); reports.add(vr); } */ // Check if we can find all the API consumer's dependencies LinkedHashSet<String> localDependencies = new LinkedHashSet<>(); for (JsonNode localDependency : configuration.get("localDependency")) { localDependencies.add(localDependency.textValue()); } if (!localDependencies.isEmpty()) { String[] jarArray = subject.libDir.list(new FileExtFilter(".jar")); if (jarArray != null) { List<String> jarFiles = Arrays.asList(jarArray); // URLs of all jars found in the lib directory for (String jar : localDependencies) { if (jarFiles.contains(jar)) { localDependencies.remove(jar); } } if (localDependencies.isEmpty()) { // all dependencies found reports.add(new VisitReport(HealthCheck.getInstance(), p, "Dependencies found", HealthCheck.NO_PROBLEM, Status.OK)); } else { VisitReport vr = new VisitReport(HealthCheck.getInstance(), p, "Dependencies missing", HealthCheck.MISSING_DEPENDENCY, Status.SEVERE); vr.setProperty("dependencies", localDependencies); vr.setProperty("directory", subject.libDir); reports.add(vr); } } } Status status = VisitReport.getWorstStatus(reports); VisitReport report = new VisitReport(HealthCheck.getInstance(), p, "API Consumer report", HealthCheck.NO_PROBLEM, status, reports); return report; }
From source file:azkaban.utils.PropsUtils.java
private static String resolveVariableReplacement(String value, Props props, LinkedHashSet<String> visitedVariables) { StringBuffer buffer = new StringBuffer(); int startIndex = 0; Matcher matcher = VARIABLE_REPLACEMENT_PATTERN.matcher(value); while (matcher.find(startIndex)) { if (startIndex < matcher.start()) { // Copy everything up front to the buffer buffer.append(value.substring(startIndex, matcher.start())); }/* w ww . j a va 2 s. c om*/ String subVariable = matcher.group(1); // Detected a cycle if (visitedVariables.contains(subVariable)) { throw new IllegalArgumentException( String.format("Circular variable substitution found: [%s] -> [%s]", StringUtils.join(visitedVariables, "->"), subVariable)); } else { // Add substitute variable and recurse. String replacement = props.get(subVariable); visitedVariables.add(subVariable); if (replacement == null) { throw new UndefinedPropertyException( String.format("Could not find variable substitution for variable(s) [%s]", StringUtils.join(visitedVariables, "->"))); } buffer.append(resolveVariableReplacement(replacement, props, visitedVariables)); visitedVariables.remove(subVariable); } startIndex = matcher.end(); } if (startIndex < value.length()) { buffer.append(value.substring(startIndex)); } return buffer.toString(); }
From source file:de._13ducks.cor.game.server.movement.SectorPathfinder.java
public static synchronized List<Node> findPath(SimplePosition start, SimplePosition target, FreePolygon startSector, MovementMap moveMap) { if (start == null || target == null) { System.out.println("FixMe: SPathfinder, irregular call: " + start + "-->" + target); return null; }/*from w ww.j av a2 s . c o m*/ FreePolygon targetSector = moveMap.containingPoly(target.x(), target.y()); if (targetSector == null) { // Ziel ungltig abbrechen System.out.println("Irregular target. Aborting"); return null; } FakeNode startNode = new FakeNode(start.x(), start.y(), startSector); Node targetNode = new FakeNode(target.x(), target.y(), targetSector); targetNode.addPolygon(targetSector); // Der Startknoten muss die Member seines Polys kennen startNode.setReachableNodes(computeDirectReachable(startNode, startSector)); // Der Zielknoten muss den Membern seines Polys bekannt sein // Die Movement-Map darf aber nicht verndert werden. Des halb mssen einige Aufrufe intern abgefangen werden und das reingedoktert werden. List<Node> preTargetNodes = Arrays.asList(computeDirectReachable(targetNode, targetSector)); PriorityBuffer open = new PriorityBuffer(); // Liste fr entdeckte Knoten LinkedHashSet<Node> containopen = new LinkedHashSet<Node>(); // Auch fr entdeckte Knoten, hiermit kann viel schneller festgestellt werden, ob ein bestimmter Knoten schon enthalten ist. LinkedHashSet<Node> closed = new LinkedHashSet<Node>(); // Liste fr fertig bearbeitete Knoten double cost_t = 0; //Movement Kosten (gerade 5, diagonal 7, wird spter festgelegt) startNode.setCost(0); //Kosten fr das Startfeld (von dem aus berechnet wird) sind natrlich 0 open.add(startNode); //Startfeld in die openlist containopen.add(startNode); targetNode.setParent(null); //"Vorgngerfeld" vom Zielfeld noch nicht bekannt for (int j = 0; j < 40000; j++) { //Anzahl der maximalen Durchlufe, bis Wegfindung aufgibt if (open.isEmpty()) { //Abbruch, wenn openlist leer ist => es gibt keinen Weg return null; } // Sortieren nicht mehr ntig, PriorityBuffer bewahrt die Felder in der Reihenfolge ihrer Priority - also dem F-Wert auf Node current = (Node) open.remove(); //der Eintrag aus der openlist mit dem niedrigesten F-Wert rausholen und gleich lschen containopen.remove(current); if (current.equals(targetNode)) { //Abbruch, weil Weg von Start nach Ziel gefunden wurde targetNode.setParent(current.getParent()); //"Vorgngerfeld" von Ziel bekannt break; } // Aus der open wurde current bereits gelscht, jetzt in die closed verschieben closed.add(current); List<Node> neighbors = computeNeighbors(current, targetNode, preTargetNodes); for (Node node : neighbors) { if (closed.contains(node)) { continue; } // Kosten dort hin berechnen cost_t = current.movementCostTo(node); if (containopen.contains(node)) { //Wenn sich der Knoten in der openlist befindet, muss berechnet werden, ob es einen krzeren Weg gibt if (current.getCost() + cost_t < node.getCost()) { //krzerer Weg gefunden? node.setCost(current.getCost() + cost_t); //-> Wegkosten neu berechnen node.setValF(node.getCost() + node.getHeuristic()); //F-Wert, besteht aus Wegkosten vom Start + Luftlinie zum Ziel node.setParent(current); //aktuelles Feld wird zum Vorgngerfeld } } else { node.setCost(current.getCost() + cost_t); node.setHeuristic(Math.sqrt(Math.pow(Math.abs((targetNode.getX() - node.getX())), 2) + Math.pow(Math.abs((targetNode.getY() - node.getY())), 2))); // geschtzte Distanz zum Ziel //Die Zahl am Ende der Berechnung ist der Aufwand der Wegsuche //5 ist schnell, 4 normal, 3 dauert lange node.setParent(current); // Parent ist die RogPosition, von dem der aktuelle entdeckt wurde node.setValF(node.getCost() + node.getHeuristic()); //F-Wert, besteht aus Wegkosten vom Start aus + Luftlinie zum Ziel open.add(node); // in openlist hinzufgen containopen.add(node); } } } if (targetNode.getParent() == null) { //kein Weg gefunden return null; } ArrayList<Node> pathrev = new ArrayList<Node>(); //Pfad aus parents erstellen, von Ziel nach Start while (!targetNode.equals(startNode)) { pathrev.add(targetNode); targetNode = targetNode.getParent(); } pathrev.add(startNode); ArrayList<Node> path = new ArrayList<Node>(); //Pfad umkehren, sodass er von Start nach Ziel ist for (int k = pathrev.size() - 1; k >= 0; k--) { path.add(pathrev.get(k)); } // Der folgende Algorithmus braucht Polygon-Infos, diese also hier einfgen startNode.addPolygon(startSector); targetNode.addPolygon(targetSector); /** * An dieser Stelle muss der Weg nocheinmal berarbeitet werden. * Es kann nmlich durch neue Tweaks sein, dass dies die Knoten nicht direkt * verbunden sind (also keinen gemeinsamen Polygon haben) * Das tritt z.B. bei der Start- und Zieleinsprungpunkt-Variierung auf. */ for (int i = 0; i < path.size() - 1; i++) { Node n1 = path.get(i); Node n2 = path.get(i + 1); FreePolygon commonSector = commonSector(n1, n2); if (commonSector == null) { // Das hier ist der interessante Fall, die beiden Knoten sind nicht direkt verbunden, es muss ein Zwischenknoten eingefgt werden: // Dessen Punkt suchen Edge direct = new Edge(n1, n2); Node newNode = null; // Die Polygone von n1 durchprobieren for (FreePolygon currentPoly : n1.getPolygons()) { List<Edge> edges = currentPoly.calcEdges(); for (Edge testedge : edges) { // Gibts da einen Schnitt? SimplePosition intersection = direct.intersectionWithEndsNotAllowed(testedge); if (intersection != null) { // Kandidat fr den nchsten Polygon FreePolygon nextPoly = null; // Kante gefunden // Von dieser Kante die Enden suchen nextPoly = getOtherPoly(testedge.getStart(), testedge.getEnd(), currentPoly); newNode = intersection.toNode(); newNode.addPolygon(currentPoly); newNode.addPolygon(nextPoly); break; } } if (newNode != null) { break; } } if (newNode == null) { // Das drfte nicht passieren, der Weg ist legal gefunden worden, muss also eigentlich existieren System.out.println("[Pathfinder][ERROR]: Cannot insert Nodes into route, aborting!"); return null; } else { path.add(i + 1, newNode); } } } return path; //Pfad zurckgeben }
From source file:com.cenrise.test.azkaban.PropsUtils.java
private static String resolveVariableReplacement(final String value, final Props props, final LinkedHashSet<String> visitedVariables) { final StringBuffer buffer = new StringBuffer(); int startIndex = 0; final Matcher matcher = VARIABLE_REPLACEMENT_PATTERN.matcher(value); while (matcher.find(startIndex)) { if (startIndex < matcher.start()) { // Copy everything up front to the buffer buffer.append(value.substring(startIndex, matcher.start())); }//from ww w. ja v a2 s . com final String subVariable = matcher.group(1); // Detected a cycle if (visitedVariables.contains(subVariable)) { throw new IllegalArgumentException( String.format("Circular variable substitution found: [%s] -> [%s]", StringUtils.join(visitedVariables, "->"), subVariable)); } else { // Add substitute variable and recurse. final String replacement = props.get(subVariable); visitedVariables.add(subVariable); if (replacement == null) { throw new UndefinedPropertyException( String.format("Could not find variable substitution for variable(s) [%s]", StringUtils.join(visitedVariables, "->"))); } buffer.append(resolveVariableReplacement(replacement, props, visitedVariables)); visitedVariables.remove(subVariable); } startIndex = matcher.end(); } if (startIndex < value.length()) { buffer.append(value.substring(startIndex)); } return buffer.toString(); }
From source file:org.pentaho.reporting.platform.plugin.ParameterXmlContentHandler.java
private Element createParameterElement(final ParameterDefinitionEntry parameter, final ParameterContext parameterContext, final Object selections) throws BeanException, ReportDataFactoryException { try {//from w ww . j a va 2 s . c o m final Element parameterElement = document.createElement("parameter"); //$NON-NLS-1$ parameterElement.setAttribute("name", parameter.getName()); //$NON-NLS-1$ final Class<?> valueType = parameter.getValueType(); parameterElement.setAttribute("type", valueType.getName()); //$NON-NLS-1$ parameterElement.setAttribute("is-mandatory", String.valueOf(parameter.isMandatory())); //$NON-NLS-1$ //$NON-NLS-2$ final String[] namespaces = parameter.getParameterAttributeNamespaces(); for (int i = 0; i < namespaces.length; i++) { final String namespace = namespaces[i]; final String[] attributeNames = parameter.getParameterAttributeNames(namespace); for (final String attributeName : attributeNames) { final String attributeValue = parameter.getParameterAttribute(namespace, attributeName, parameterContext); // expecting: label, parameter-render-type, parameter-layout // but others possible as well, so we set them all final Element attributeElement = document.createElement("attribute"); // NON-NLS attributeElement.setAttribute("namespace", namespace); // NON-NLS attributeElement.setAttribute("name", attributeName); // NON-NLS attributeElement.setAttribute("value", attributeValue); // NON-NLS parameterElement.appendChild(attributeElement); } } final Class<?> elementValueType; if (valueType.isArray()) { elementValueType = valueType.getComponentType(); } else { elementValueType = valueType; } if (Date.class.isAssignableFrom(elementValueType)) { parameterElement.setAttribute("timzone-hint", computeTimeZoneHint(parameter, parameterContext));//$NON-NLS-1$ } final LinkedHashSet<Object> selectionSet = new LinkedHashSet<Object>(); if (selections != null) { if (selections.getClass().isArray()) { final int length = Array.getLength(selections); for (int i = 0; i < length; i++) { final Object value = Array.get(selections, i); selectionSet.add(resolveSelectionValue(value)); } } else { selectionSet.add(resolveSelectionValue(selections)); } } else { final String type = parameter.getParameterAttribute(ParameterAttributeNames.Core.NAMESPACE, ParameterAttributeNames.Core.TYPE, parameterContext); if (ParameterAttributeNames.Core.TYPE_DATEPICKER.equals(type) && Date.class.isAssignableFrom(valueType)) { if (isGenerateDefaultDates()) { selectionSet.add(new Date()); } } } @SuppressWarnings("rawtypes") final LinkedHashSet handledValues = (LinkedHashSet) selectionSet.clone(); if (parameter instanceof ListParameter) { final ListParameter asListParam = (ListParameter) parameter; parameterElement.setAttribute("is-multi-select", //$NON-NLS-1$ String.valueOf(asListParam.isAllowMultiSelection())); //$NON-NLS-2$ parameterElement.setAttribute("is-strict", String.valueOf(asListParam.isStrictValueCheck())); //$NON-NLS-1$ //$NON-NLS-2$ parameterElement.setAttribute("is-list", "true"); //$NON-NLS-1$ //$NON-NLS-2$ final Element valuesElement = document.createElement("values"); //$NON-NLS-1$ parameterElement.appendChild(valuesElement); final ParameterValues possibleValues = asListParam.getValues(parameterContext); for (int i = 0; i < possibleValues.getRowCount(); i++) { final Object key = possibleValues.getKeyValue(i); final Object value = possibleValues.getTextValue(i); final Element valueElement = document.createElement("value"); //$NON-NLS-1$ valuesElement.appendChild(valueElement); valueElement.setAttribute("label", String.valueOf(value)); //$NON-NLS-1$ //$NON-NLS-2$ valueElement.setAttribute("type", elementValueType.getName()); //$NON-NLS-1$ if (key instanceof Number) { final BigDecimal bd = new BigDecimal(String.valueOf(key)); valueElement.setAttribute("selected", String.valueOf(selectionSet.contains(bd)));//$NON-NLS-1$ handledValues.remove(bd); } else if (key == null) { if (selections == null || selectionSet.contains(null)) { valueElement.setAttribute("selected", "true");//$NON-NLS-1$ handledValues.remove(null); } } else { valueElement.setAttribute("selected", String.valueOf(selectionSet.contains(key)));//$NON-NLS-1$ handledValues.remove(key); } if (key == null) { valueElement.setAttribute("null", "true"); //$NON-NLS-1$ //$NON-NLS-2$ } else { valueElement.setAttribute("null", "false"); //$NON-NLS-1$ //$NON-NLS-2$ valueElement.setAttribute("value", convertParameterValueToString(parameter, parameterContext, key, elementValueType)); //$NON-NLS-1$ //$NON-NLS-2$ } } // Only add invalid values to the selection list for non-strict parameters if (!asListParam.isStrictValueCheck()) { for (final Object key : handledValues) { final Element valueElement = document.createElement("value"); //$NON-NLS-1$ valuesElement.appendChild(valueElement); valueElement.setAttribute("label", Messages.getInstance() //$NON-NLS-1$ .getString("ReportPlugin.autoParameter", String.valueOf(key))); //$NON-NLS-1$ valueElement.setAttribute("type", elementValueType.getName()); //$NON-NLS-1$ if (key instanceof Number) { BigDecimal bd = new BigDecimal(String.valueOf(key)); valueElement.setAttribute("selected", String.valueOf(selectionSet.contains(bd)));//$NON-NLS-1$ } else { valueElement.setAttribute("selected", String.valueOf(selectionSet.contains(key)));//$NON-NLS-1$ } if (key == null) { valueElement.setAttribute("null", "true"); //$NON-NLS-1$ //$NON-NLS-2$ } else { valueElement.setAttribute("null", "false"); //$NON-NLS-1$ //$NON-NLS-2$ valueElement.setAttribute("value", convertParameterValueToString(parameter, parameterContext, key, elementValueType)); //$NON-NLS-1$ //$NON-NLS-2$ } } } } else if (parameter instanceof PlainParameter) { // apply defaults, this is the easy case parameterElement.setAttribute("is-multi-select", "false"); //$NON-NLS-1$ //$NON-NLS-2$ parameterElement.setAttribute("is-strict", "false"); //$NON-NLS-1$ //$NON-NLS-2$ parameterElement.setAttribute("is-list", "false"); //$NON-NLS-1$ //$NON-NLS-2$ if (selections != null) { final Element valuesElement = document.createElement("values"); //$NON-NLS-1$ parameterElement.appendChild(valuesElement); final Element valueElement = document.createElement("value"); //$NON-NLS-1$ valuesElement.appendChild(valueElement); valueElement.setAttribute("type", valueType.getName()); //$NON-NLS-1$ valueElement.setAttribute("selected", "true");//$NON-NLS-1$ valueElement.setAttribute("null", "false"); //$NON-NLS-1$ //$NON-NLS-2$ final String value = convertParameterValueToString(parameter, parameterContext, selections, valueType); valueElement.setAttribute("value", value); //$NON-NLS-1$ //$NON-NLS-2$ valueElement.setAttribute("label", value); //$NON-NLS-1$ //$NON-NLS-2$ } } return parameterElement; } catch (BeanException be) { logger.error(Messages.getInstance().getString("ReportPlugin.errorFailedToGenerateParameter", parameter.getName(), String.valueOf(selections)), be); throw be; } }
From source file:org.alfresco.solr.SolrInformationServer.java
private void updateDescendantDocs(NodeMetaData parentNodeMetaData, boolean overwrite, SolrQueryRequest request, UpdateRequestProcessor processor, LinkedHashSet<Long> stack) throws AuthenticationException, IOException, JSONException { if (stack.contains(parentNodeMetaData.getId())) { log.warn("Found descendant data loop for node id " + parentNodeMetaData.getId()); log.warn("... stack to node =" + stack); return;/*from w ww . ja va 2 s. c o m*/ } else { try { stack.add(parentNodeMetaData.getId()); doUpdateDescendantDocs(parentNodeMetaData, overwrite, request, processor, stack); } finally { stack.remove(parentNodeMetaData.getId()); } } }
From source file:com.gtwm.pb.model.manageData.DataManagement.java
public int importCSV(HttpServletRequest request, TableInfo table, boolean updateExistingRecords, BaseField recordIdentifierField, boolean generateRowIds, char separator, char quotechar, int numHeaderLines, boolean useRelationDisplayValues, boolean importSequenceValues, boolean requireExactRelationMatches, boolean trim, boolean merge, List<FileItem> multipartItems, String csvContent) throws SQLException, InputRecordException, IOException, CantDoThatException, ObjectNotFoundException, DisallowedException, CodingErrorException { if (!FileUpload.isMultipartContent(new ServletRequestContext(request))) { if (csvContent == null) { throw new CantDoThatException( "To import CSV content, a file must be uploaded (form posted as multi-part) or csv_content specified"); }/*from w w w. j a v a2 s.c o m*/ } int numImportedRecords = 0; // get field set to import into. LinkedHashSet to ensure order is // retained so the right values are imported into the right fields LinkedHashSet<BaseField> fields = new LinkedHashSet<BaseField>(table.getFields()); // if row IDs aren't included in the data to import, remove ID from the // field set BaseField primaryKey = table.getPrimaryKey(); if (recordIdentifierField == null) { recordIdentifierField = primaryKey; } if (generateRowIds || (updateExistingRecords && !recordIdentifierField.equals(primaryKey))) { fields.remove(primaryKey); } Map<RelationField, Map<String, String>> relationLookups = new HashMap<RelationField, Map<String, String>>(); // Remove fields which shouldn't be modified during the import // For serial fields, if we need to set serial values explicitly, this // will have to be dealt with later for (BaseField field : table.getFields()) { if (field instanceof SequenceField && (!field.equals(primaryKey)) && (!importSequenceValues)) { fields.remove(field); } else if (field.getHidden()) { if (field.getFieldName().equals(HiddenFields.VIEW_COUNT.getFieldName()) || field.getFieldName().equals(HiddenFields.COMMENTS_FEED.getFieldName())) { fields.remove(field); } else if (updateExistingRecords) { if (field.getFieldName().equals(HiddenFields.DATE_CREATED.getFieldName()) || field.getFieldName().equals(HiddenFields.CREATED_BY.getFieldName())) { fields.remove(field); } } } else if (!field.getFieldCategory().savesData()) { fields.remove(field); } // Also, if importing relations by display value, look up // display/internal value mappings if (useRelationDisplayValues && field instanceof RelationField) { Map<String, String> displayToInternalValue = ((RelationFieldDefn) field).getItems(true, false); relationLookups.put((RelationField) field, displayToInternalValue); } } // Prepare SQL String insertSQLCode = null; String updateSQLCode = null; String logCreationSQLCode = null; // If updating, we'll need a record ID value. Depending on what the // identifier field is, this could be one of a couple of different types String recordIdentifierString = null; Integer recordIdentifierInteger = null; int recordIdentifierFieldNum = 0; DatabaseFieldType identifierFieldDbType = null; if (updateExistingRecords) { identifierFieldDbType = recordIdentifierField.getDbType(); if (!identifierFieldDbType.equals(DatabaseFieldType.VARCHAR) && !identifierFieldDbType.equals(DatabaseFieldType.INTEGER) && !identifierFieldDbType.equals(DatabaseFieldType.SERIAL)) { throw new CantDoThatException("The record identifier field has to be text or a whole number, " + recordIdentifierField + " is a " + identifierFieldDbType); } updateSQLCode = "UPDATE " + table.getInternalTableName() + " SET "; int fieldNum = 0; for (BaseField field : fields) { fieldNum += 1; if (merge) { // Update database only if there's a non-null value from the // spreadsheet updateSQLCode += field.getInternalFieldName() + " = COALESCE(?," + field.getInternalFieldName() + "), "; } else { updateSQLCode += field.getInternalFieldName() + " = ?, "; } if (field.equals(recordIdentifierField)) { recordIdentifierFieldNum = fieldNum; } } if (recordIdentifierFieldNum == 0) { throw new CantDoThatException("Can't find the field specified as record identifier (" + recordIdentifierField + ") in the list of table fields " + fields + " in table " + table); } updateSQLCode = updateSQLCode.substring(0, updateSQLCode.length() - 2); updateSQLCode += " WHERE " + recordIdentifierField.getInternalFieldName() + "=?"; logCreationSQLCode = "UPDATE " + table.getInternalTableName() + " SET " + table.getField(HiddenFields.DATE_CREATED.getFieldName()).getInternalFieldName() + "=?, " + table.getField(HiddenFields.CREATED_BY.getFieldName()).getInternalFieldName() + "=? WHERE " + primaryKey.getInternalFieldName() + "=?"; } insertSQLCode = "INSERT INTO " + table.getInternalTableName() + "("; String placeholders = ""; for (BaseField field : fields) { insertSQLCode += field.getInternalFieldName() + ", "; placeholders += "?, "; } placeholders = placeholders.substring(0, placeholders.length() - 2); insertSQLCode = insertSQLCode.substring(0, insertSQLCode.length() - 2) + ") VALUES (" + placeholders + ")"; // Find content to import Reader inputStreamReader = null; if (csvContent != null) { inputStreamReader = new StringReader(csvContent); } else { for (FileItem item : multipartItems) { // if item is a file if (!item.isFormField()) { if (item.getName().toLowerCase().endsWith(".xls")) { throw new CantDoThatException( "You need to upload as a CSV to import, Excel files can't be imported directly"); } inputStreamReader = new InputStreamReader(item.getInputStream()); break; } } } if (inputStreamReader == null) { throw new CantDoThatException("No file uploaded"); } CSVReader csvReader = new CSVReader(inputStreamReader, separator, quotechar, numHeaderLines); // returns a list of String arrays List<String[]> csvLines = (List<String[]>) csvReader.readAll(); // do db inserts Connection conn = null; PreparedStatement statement = null; // backupInsertStatement is for when an update returns 0 rows affected, // i.e. there's no matching row. In this case, do an insert PreparedStatement backupInsertStatement = null; PreparedStatement logCreationStatement = null; // These two variables used in exception handling int importLine = 0; BaseField fieldImported = null; Timestamp importTime = new Timestamp(System.currentTimeMillis()); AppUserInfo loggedInUser = authManager.getUserByUserName(request, request.getRemoteUser()); String fullname = loggedInUser.getForename() + " " + loggedInUser.getSurname() + " (" + loggedInUser.getUserName() + ")"; try { conn = this.dataSource.getConnection(); conn.setAutoCommit(false); if (updateExistingRecords) { statement = conn.prepareStatement(updateSQLCode); backupInsertStatement = conn.prepareStatement(insertSQLCode); logCreationStatement = conn.prepareStatement(logCreationSQLCode); } else { statement = conn.prepareStatement(insertSQLCode); } CSVLINE: for (String[] csvLineArray : csvLines) { // convert to an object rather than a primitive array - // easier to work with List<String> lineValues = Arrays.asList(csvLineArray); importLine++; // skip blank lines if (lineValues.size() == 1) { if (lineValues.get(0).length() == 0) { continue CSVLINE; } } int fieldNum = 0; for (BaseField field : fields) { fieldImported = field; fieldNum++; if (field.getHidden()) { String fieldName = field.getFieldName(); if (fieldName.equals(HiddenFields.LOCKED.getFieldName())) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else if (fieldName.equals(HiddenFields.DATE_CREATED.getFieldName()) || fieldName.equals(HiddenFields.LAST_MODIFIED.getFieldName())) { statement.setTimestamp(fieldNum, importTime); if (updateExistingRecords) { backupInsertStatement.setTimestamp(fieldNum, importTime); } } else if (fieldName.equals(HiddenFields.CREATED_BY.getFieldName()) || fieldName.equals(HiddenFields.MODIFIED_BY.getFieldName())) { statement.setString(fieldNum, fullname); if (updateExistingRecords) { backupInsertStatement.setString(fieldNum, fullname); } } } else if (fieldNum > lineValues.size()) { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } else { String lineValue = lineValues.get(fieldNum - 1); if (lineValue != null) { if (trim) { lineValue = lineValue.trim(); } if (lineValue.equals("")) { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } else { if ((field instanceof FileField) && (generateRowIds)) { throw new CantDoThatException( "Cannot generate row ids when importing file names. See line " + importLine + ", field '" + field.getFieldName() + "' with value '" + lineValue + "'"); } switch (field.getDbType()) { case VARCHAR: statement.setString(fieldNum, lineValue); if (updateExistingRecords) { backupInsertStatement.setString(fieldNum, lineValue); if (field.equals(recordIdentifierField)) { recordIdentifierString = lineValue; } } break; case TIMESTAMP: // deal with month and year // resolution dates exported if (lineValue.matches("^[a-zA-Z]{3}\\s\\d{2,4}$")) { lineValue = "01 " + lineValue; } else if (lineValue.matches("^\\d{2,4}")) { lineValue = "01 Jan " + lineValue; } try { Calendar calValue = CalendarParser.parse(lineValue, CalendarParser.DD_MM_YY); statement.setTimestamp(fieldNum, new Timestamp(calValue.getTimeInMillis())); if (updateExistingRecords) { backupInsertStatement.setTimestamp(fieldNum, new Timestamp(calValue.getTimeInMillis())); } } catch (CalendarParserException cpex) { throw new InputRecordException("Error importing line " + importLine + ", field " + field + ": " + cpex.getMessage(), field, cpex); } break; case FLOAT: lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); statement.setDouble(fieldNum, Double.valueOf(lineValue)); if (updateExistingRecords) { backupInsertStatement.setDouble(fieldNum, Double.valueOf(lineValue)); } break; case INTEGER: if ((field instanceof RelationField) && (useRelationDisplayValues)) { // find key value for display value RelationField relationField = (RelationField) field; Map<String, String> valueKeyMap = relationLookups.get(relationField); String internalValueString = valueKeyMap.get(lineValue); if (internalValueString == null) { if (!requireExactRelationMatches) { // A very basic fuzzy matching // algorithm String potentialDisplayValue = null; String lineValueLowerCase = lineValue.toLowerCase(); FUZZYMATCH: for (Map.Entry<String, String> entry : valueKeyMap .entrySet()) { potentialDisplayValue = entry.getKey(); if (potentialDisplayValue.toLowerCase() .contains(lineValueLowerCase)) { internalValueString = entry.getValue(); break FUZZYMATCH; } } } if (internalValueString == null) { throw new CantDoThatException("Error importing line " + importLine + ", field " + relationField + ": Can't find a related '" + relationField.getRelatedTable() + "' for " + relationField.getDisplayField() + " '" + lineValue + "'. "); } } int keyValue = Integer.valueOf(internalValueString); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } } else { lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); int keyValue = Integer.valueOf(lineValue); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } } break; case SERIAL: lineValue = lineValue.trim().replaceAll("[^\\d\\.\\+\\-eE]", ""); int keyValue = Integer.valueOf(lineValue); statement.setInt(fieldNum, keyValue); if (updateExistingRecords) { backupInsertStatement.setInt(fieldNum, keyValue); if (field.equals(recordIdentifierField)) { recordIdentifierInteger = keyValue; } } break; case BOOLEAN: boolean filterValueIsTrue = Helpers.valueRepresentsBooleanTrue(lineValue); statement.setBoolean(fieldNum, filterValueIsTrue); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, filterValueIsTrue); } break; } } } else { // booleans have a not null constraint if (field.getDbType().equals(Types.BOOLEAN)) { statement.setBoolean(fieldNum, false); if (updateExistingRecords) { backupInsertStatement.setBoolean(fieldNum, false); } } else { statement.setNull(fieldNum, Types.NULL); if (updateExistingRecords) { backupInsertStatement.setNull(fieldNum, Types.NULL); } } } } } if (updateExistingRecords) { // for potential error messages String recordIdentifierDescription = null; if (identifierFieldDbType.equals(DatabaseFieldType.INTEGER) || identifierFieldDbType.equals(DatabaseFieldType.SERIAL)) { if (recordIdentifierInteger == null) { throw new InputRecordException( "Can't find a record identifier value at line " + importLine, recordIdentifierField); } recordIdentifierDescription = recordIdentifierField.getFieldName() + " = " + recordIdentifierInteger; // Set the 'WHERE recordIdentifier = ?' clause statement.setInt(fields.size() + 1, recordIdentifierInteger); } else { if (recordIdentifierString == null) { throw new InputRecordException( "Can't find a record identifier value at line " + importLine, recordIdentifierField); } recordIdentifierDescription = recordIdentifierField.getFieldName() + " = '" + recordIdentifierString + "'"; // Set the 'WHERE recordIdentifier = ?' clause statement.setString(fields.size() + 1, recordIdentifierString); } int rowsAffected = statement.executeUpdate(); if (rowsAffected == 0) { // If can't find a match to update, insert a record // instead backupInsertStatement.executeUpdate(); // NB Postgres specific code to find Row ID of newly // inserted record, not cross-db compatible String newRowIdSQLCode = "SELECT currval('" + table.getInternalTableName() + "_" + primaryKey.getInternalFieldName() + "_seq')"; PreparedStatement newRowIdStatement = conn.prepareStatement(newRowIdSQLCode); ResultSet newRowIdResults = newRowIdStatement.executeQuery(); if (newRowIdResults.next()) { int newRowId = newRowIdResults.getInt(1); // Add creation metadata to the new row logCreationStatement.setTimestamp(1, importTime); logCreationStatement.setString(2, fullname); logCreationStatement.setInt(3, newRowId); int creationLogRowsAffected = logCreationStatement.executeUpdate(); if (creationLogRowsAffected == 0) { throw new SQLException( "Unable to update creation metadata of newly inserted record, using query " + logCreationStatement); } } else { newRowIdResults.close(); newRowIdStatement.close(); throw new SQLException("Row ID not found for the newly inserted record. '" + newRowIdStatement + "' didn't work"); } newRowIdResults.close(); newRowIdStatement.close(); } else if (rowsAffected > 1) { throw new InputRecordException("Error importing line " + importLine + ". The record identifier field " + recordIdentifierDescription + " should match only 1 record in the database but it actually matches " + rowsAffected, recordIdentifierField); } // reset to null for the next line recordIdentifierString = null; recordIdentifierInteger = null; } else { statement.executeUpdate(); } numImportedRecords += 1; } statement.close(); if (backupInsertStatement != null) { backupInsertStatement.close(); } if (logCreationStatement != null) { logCreationStatement.close(); } // reset the primary key ID sequence so new records can be added resetSequence((SequenceField) primaryKey, conn); // and any other sequence fields if (importSequenceValues) { for (BaseField field : table.getFields()) { if ((!field.equals(primaryKey)) && field instanceof SequenceField) { resetSequence((SequenceField) field, conn); } } } // ANALYZE the table after import if (numImportedRecords > 1000) { Statement analyzeStatement = conn.createStatement(); analyzeStatement.execute("ANALYZE " + table.getInternalTableName()); analyzeStatement.close(); } conn.commit(); } catch (SQLException sqlex) { String databaseErrorMessage = Helpers.replaceInternalNames(sqlex.getMessage(), table.getDefaultReport()); logger.warn("Import failed, statement is " + statement); logger.warn("Backup insert statement is " + backupInsertStatement); String errorMessage = "Error importing CSV line " + importLine; if (!fieldImported.getHidden()) { errorMessage += ", field '" + fieldImported + "'"; } errorMessage += ": " + databaseErrorMessage; throw new InputRecordException(errorMessage, fieldImported, sqlex); } catch (NumberFormatException nfex) { String causeMessage = nfex.getMessage(); causeMessage = causeMessage.replaceAll("For input string", "value"); String errorMessage = "Error parsing number when importing CSV line " + importLine; if (!fieldImported.getHidden()) { errorMessage += ", field '" + fieldImported + "'"; } errorMessage += ": " + causeMessage; throw new InputRecordException(errorMessage, fieldImported, nfex); } finally { if (conn != null) { conn.close(); } } this.logLastDataChangeTime(request); logLastTableDataChangeTime(table); UsageLogger usageLogger = new UsageLogger(this.dataSource); String logMessage = "" + numImportedRecords; if (updateExistingRecords) { logMessage += " records imported"; } else { logMessage += " new records imported"; } if (csvContent != null) { logMessage += " from file"; } usageLogger.logDataChange(loggedInUser, table, null, AppAction.CSV_IMPORT, -1, logMessage); UsageLogger.startLoggingThread(usageLogger); return numImportedRecords; }