List of usage examples for org.apache.commons.csv CSVRecord getRecordNumber
public long getRecordNumber()
From source file:nl.paston.bonita.importfile.Main.java
protected static Map<String, Serializable> parseRecord(CSVRecord record, CSVRecord fullHeader) { if (record == null) { log.warn("Record is null."); return null; }/*from w w w . j a va 2s . co m*/ log.info("Parsing record number: " + (record.getRecordNumber() - 1)); log.debug(" with content: " + record.toString()); final Map<String, Serializable> map = new HashMap<>(); for (int i = 0; i < record.size(); i++) { String headerFieldType = getHeaderFieldType(fullHeader.get(i)); Object recordField = getRecordField(headerFieldType, record.get(i)); if (recordField != null) { String headerField = getHeaderField(fullHeader.get(i)); String[] headerFieldParts = headerField.split("\\."); Map<String, Serializable> targetMap = map; for (int j = 0; j < headerFieldParts.length - 1; j++) { try { targetMap = (Map<String, Serializable>) targetMap.computeIfAbsent(headerFieldParts[j], x -> new HashMap<>()); } catch (ClassCastException ex) { log.debug("Problem parsing: " + headerFieldParts[j]); } } String currentHeaderField = headerFieldParts[headerFieldParts.length - 1]; Matcher listMatcher = Pattern.compile("\\[(.*)\\]").matcher(currentHeaderField); if (listMatcher.find()) { String parametersString = listMatcher.group(1); Matcher chfNameMatcher = Pattern.compile("(.*)\\[").matcher(currentHeaderField); if (chfNameMatcher.find()) { String chfName = chfNameMatcher.group(1); String[] parameters = parametersString.split("&"); List<Serializable> list = (List<Serializable>) targetMap.computeIfAbsent(chfName, x -> new ArrayList<>()); if (parametersString.isEmpty()) { list.add((Serializable) recordField); } else { Map<String, Serializable> subMap = new HashMap<>(); for (String parameter : parameters) { String[] parameterKeys = parameter.split("="); if (parameterKeys.length == 2) { subMap.put(parameterKeys[0], parameterKeys[1]); } else if (parameterKeys.length == 1) { subMap.put(parameterKeys[0], (Serializable) recordField); } else { log.error("Wrong number of parameters for: " + currentHeaderField); System.exit(1); } } list.add((Serializable) subMap); } } } else { targetMap.put(currentHeaderField, (Serializable) recordField); } } else { log.warn("Skipped value for record item: " + record.get(i)); } } return map; }
From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensIdConverter.java
/** * Loads the mapping file./*from w ww. j av a 2s . c o m*/ * * <p> * The columns of the mapping file are: * <ol> * <li>MovieLens id of the movie</li> * <li>rating</li> * <li>average</li> * <li>IMDb id of the movie</li> * <li>title and year</li> * </ol> * * @param mappingFilepath the file that contains the mapping */ public MovieLensIdConverter(String mappingFilepath) throws IOException { checkArgument(new File(mappingFilepath).exists(), "The mapping file must exist."); CSVParser parser = new CSVParser( new CleanRatingsReader(new BufferedReader(new FileReader(mappingFilepath))), CSVFormat.MYSQL); mappings = new HashMap<>(); for (CSVRecord record : parser) { if (record.size() != RECORD_SIZE) { parser.close(); throw new IllegalStateException("Error: unable to parse the movie file \"" + mappingFilepath + "\". A list of five tab separated values is expected. Approximate" + " line number: " + record.getRecordNumber()); } mappings.put(record.get(MOVIELENS_MOVIE_ID_INDEX), record.get(IMDB_MOVIE_ID_INDEX)); } parser.close(); }
From source file:nzilbb.csv.CsvDeserializer.java
/** * Deserializes the serialized data, generating one or more {@link Graph}s. * <p>Many data formats will only yield one graph (e.g. Transcriber * transcript or Praat textgrid), however there are formats that * are capable of storing multiple transcripts in the same file * (e.g. AGTK, Transana XML export), which is why this method * returns a list.//from w w w . j a v a 2 s.c o m * <p>This deserializer generates one graph per data row in the CSV file. * @return A list of valid (if incomplete) {@link Graph}s. * @throws SerializerNotConfiguredException if the object has not been configured. * @throws SerializationParametersMissingException if the parameters for this particular graph have not been set. * @throws SerializationException if errors occur during deserialization. */ public Graph[] deserialize() throws SerializerNotConfiguredException, SerializationParametersMissingException, SerializationException { if (participantLayer == null) throw new SerializerNotConfiguredException("Participant layer not set"); if (turnLayer == null) throw new SerializerNotConfiguredException("Turn layer not set"); if (utteranceLayer == null) throw new SerializerNotConfiguredException("Utterance layer not set"); if (wordLayer == null) throw new SerializerNotConfiguredException("Word layer not set"); if (schema == null) throw new SerializerNotConfiguredException("Layer schema not set"); validate(); String participantColumn = (String) parameters.get("who").getValue(); String textColumn = (String) parameters.get("text").getValue(); // if there are errors, accumlate as many as we can before throwing SerializationException SerializationException errors = null; Vector<Graph> graphs = new Vector<Graph>(); Iterator<CSVRecord> records = getParser().iterator(); while (records.hasNext()) { CSVRecord record = records.next(); Graph graph = new Graph(); if (parameters == null || parameters.get("id") == null || parameters.get("id").getValue() == null) { graph.setId(getName() + "-" + record.getRecordNumber()); } else { graph.setId(record.get((String) parameters.get("id").getValue())); } graph.setOffsetUnits(Constants.UNIT_CHARACTERS); // creat the 0 anchor to prevent graph tagging from creating one with no confidence Anchor firstAnchor = graph.getOrCreateAnchorAt(0.0, Constants.CONFIDENCE_MANUAL); Anchor lastAnchor = firstAnchor; // add layers to the graph // we don't just copy the whole schema, because that would imply that all the extra layers // contained no annotations, which is not necessarily true graph.addLayer((Layer) participantLayer.clone()); graph.getSchema().setParticipantLayerId(participantLayer.getId()); graph.addLayer((Layer) turnLayer.clone()); graph.getSchema().setTurnLayerId(turnLayer.getId()); graph.addLayer((Layer) utteranceLayer.clone()); graph.getSchema().setUtteranceLayerId(utteranceLayer.getId()); graph.addLayer((Layer) wordLayer.clone()); graph.getSchema().setWordLayerId(wordLayer.getId()); if (parameters != null) { for (Parameter p : parameters.values()) { if (p.getValue() instanceof Layer) { Layer layer = (Layer) p.getValue(); if (layer != null && graph.getLayer(layer.getId()) == null) { // haven't added this layer yet graph.addLayer((Layer) layer.clone()); } } } } // participant/author Annotation participant = graph.createTag(graph, schema.getParticipantLayerId(), record.get(participantColumn)); // meta-data for (String header : getHeaderMap().keySet()) { if (header.trim().length() == 0) continue; Parameter p = parameters.get("header_" + getHeaderMap().get(header)); if (p != null && p.getValue() != null) { Layer layer = (Layer) p.getValue(); String value = record.get(header); if (layer.getParentId().equals(schema.getRoot().getId())) // graph tag { graph.createTag(graph, layer.getId(), value); } else // participant tag { graph.createTag(participant, layer.getId(), value); } } // parameter set } // next header // text Annotation turn = new Annotation(null, participant.getLabel(), getTurnLayer().getId()); graph.addAnnotation(turn); turn.setParent(participant); turn.setStart(graph.getOrCreateAnchorAt(0.0, Constants.CONFIDENCE_MANUAL)); Annotation line = new Annotation(null, turn.getLabel(), getUtteranceLayer().getId()); line.setParentId(turn.getId()); line.setStart(turn.getStart()); int iLastPosition = 0; String sLine = record.get(textColumn).trim(); int iNumChars = sLine.length(); line = new Annotation(null, sLine, getUtteranceLayer().getId()); line.setParentId(turn.getId()); line.setStart(turn.getStart()); Anchor end = graph.getOrCreateAnchorAt(((double) iNumChars + 1), Constants.CONFIDENCE_MANUAL); line.setEnd(end); graph.addAnnotation(line); // ensure we have an utterance tokenizer if (getTokenizer() == null) { setTokenizer(new SimpleTokenizer(getUtteranceLayer().getId(), getWordLayer().getId())); } try { tokenizer.transform(graph); } catch (TransformationException exception) { if (errors == null) errors = new SerializationException(); if (errors.getCause() == null) errors.initCause(exception); errors.addError(SerializationException.ErrorType.Tokenization, exception.getMessage()); } graph.commit(); OrthographyClumper clumper = new OrthographyClumper(wordLayer.getId(), utteranceLayer.getId()); try { // clump non-orthographic 'words' with real words clumper.transform(graph); graph.commit(); } catch (TransformationException exception) { if (errors == null) errors = new SerializationException(); if (errors.getCause() == null) errors.initCause(exception); errors.addError(SerializationException.ErrorType.Tokenization, exception.getMessage()); } if (errors != null) throw errors; // set end anchors of graph tags for (Annotation a : graph.list(getParticipantLayer().getId())) { a.setStartId(firstAnchor.getId()); a.setEndId(lastAnchor.getId()); } graph.commit(); graphs.add(graph); } // next record return graphs.toArray(new Graph[0]); }
From source file:org.apache.nifi.processors.ParseCSV.ParseCSV.java
@Override public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { final Charset charset = Charset.defaultCharset(); FlowFile flowFile = session.get();// w ww. j a v a 2s.c o m if (flowFile == null) { return; } // TODO implement final Map<String, String> attributes = new LinkedHashMap<>(); final String format = context.getProperty(FORMAT).getValue(); final boolean create_attributes = Boolean.parseBoolean(context.getProperty(CREATE_ATTRIBUTES).getValue()); final char delimiter = context.getProperty(DELIMITER).getValue().charAt(0); final boolean with_header = Boolean.parseBoolean(context.getProperty(WITH_HEADER).getValue()); final String output_format = context.getProperty(OUTPUT_FORMAT).getValue(); final String custom_header = context.getProperty(CUSTOM_HEADER).getValue(); final String column_mask = context.getProperty(COLUMN_MASK).getValue(); final String column_encrypt = context.getProperty(COLUMN_ENCRYPT).getValue(); final String column_tokenize = context.getProperty(COLUMN_TOKENIZE).getValue(); final String tokenize_unique_identifier = context.getProperty(TOKENIZE_UNQIUE_IDENTIFIER).getValue(); final String tokenized_ouput = context.getProperty(TOKENIZED_OUTPUT).getValue(); final String encryptionKey = "Bar12345Bar12345"; final String static_schema = context.getProperty(STATIC_SCHEMA).getValue(); // new flowfile here final org.apache.nifi.util.ObjectHolder<FlowFile> holder = new org.apache.nifi.util.ObjectHolder<>(null); flowFile = session.write(flowFile, new StreamCallback() { @Override public void process(InputStream inputStream, OutputStream outputStream) throws IOException { CSVFormat csvFormat = buildFormat(format, delimiter, with_header, custom_header); CSVParser csvParser = new CSVParser(new InputStreamReader(inputStream, charset), csvFormat); CSVPrinter csvPrinter = new CSVPrinter(new OutputStreamWriter(outputStream, charset), csvFormat); String headerArray[]; ArrayList<String> columnMaskList = new ArrayList<>(); ArrayList<String> columnEncryptList = new ArrayList<String>(); ArrayList<String> columnTokenizeList = new ArrayList<String>(); List<String> maskValueHolder = new LinkedList<>(); FlowFile tokenized = session.create(); // print header if needed if (custom_header != null && output_format.equals("CSV") && static_schema == null) { csvPrinter.printRecord(custom_header); headerArray = custom_header.split(","); } else if (static_schema != null && custom_header == null) { csvPrinter.printRecord(static_schema.replace("\"", "")); headerArray = static_schema.split(","); } else { headerArray = csvParser.getHeaderMap().keySet().toArray(new String[0]); csvPrinter.printRecord(headerArray); } if (column_mask != null) { columnMaskList = new ArrayList<>(Arrays.asList(column_mask.replace("\"", "").split(","))); } if (column_encrypt != null) { columnEncryptList = new ArrayList<>(Arrays.asList(column_encrypt.split(","))); } if (column_tokenize != null) { columnTokenizeList = new ArrayList<>(Arrays.asList(column_tokenize.split(","))); } // loop through records and print for (final CSVRecord record : csvParser) { Map<String, String> k = record.toMap(); for (Map.Entry<String, String> konj : k.entrySet()) { //System.out.println(konj.getValue()); } // generate attributes if required per record if (create_attributes) { for (int i = 0; i < headerArray.length; i++) { //attributes.put(headerArray[i], record.get(i)); attributes.put(headerArray[i] + "." + record.getRecordNumber(), record.get(i)); } } // check masked columns if (column_mask != null || column_encrypt != null) { // we have to loop through the header array and match user requested mask columns for (int i = 0; i < headerArray.length; i++) { //System.out.println(headerArray[i] + "." + record.getRecordNumber() + " - " + mask(record.get(i))); if (columnMaskList.contains(headerArray[i])) { // set mask maskValueHolder.add(mask(record.get(i))); // construct tokenization row for external DB store if (columnTokenizeList.contains(headerArray[i])) { final String tokenizedRow; tokenizedRow = tokenizationOut(tokenized_ouput, headerArray[i], tokenize_unique_identifier, mask(record.get(i)), record.get(i), Long.toString(record.getRecordNumber())); tokenized = session.append(tokenized, new OutputStreamCallback() { @Override public void process(OutputStream outputStream) throws IOException { outputStream.write(tokenizedRow.getBytes()); } }); } } else if (columnEncryptList.contains(headerArray[i])) { // encrypt maskValueHolder.add(new String(Encrypt(record.get(i), encryptionKey), "UTF-8")); } else { // no mask maskValueHolder.add(record.get(i)); } } csvPrinter.printRecord(maskValueHolder); // clear mask column holder maskValueHolder.clear(); } else { // no masking or encryption required, print record switch (output_format) { case "CSV": //csvPrinter.printRecord(record); List<String> items = Arrays.asList(static_schema.split(",")); String lastColumn = items.get(items.size() - 1); String test = ""; for (String item : items) { if (item != lastColumn) { test += record.get(item) + ","; } else { test += record.get(item); } } csvPrinter.printRecord(test.replace("^\"|\"$", "")); break; case "JSON": String json = new ObjectMapper().writer().withDefaultPrettyPrinter() .writeValueAsString(record.toMap()) + "\n"; if (json.length() > 0) { outputStream.write(json.getBytes()); } //List<Map<?, ?>> data = readObjectsFromCsv(inputStream); //String adis = writeAsJson(data); //outputStream.write(writeAsJson(data).getBytes()); break; case "XML": outputStream.write(new XmlMapper().writeValueAsString(record.toMap()).getBytes()); break; } } } csvPrinter.flush(); csvPrinter.close(); holder.set(tokenized); } }); flowFile = session.putAllAttributes(flowFile, attributes); session.transfer(flowFile, RELATIONSHIP_SUCCESS); session.transfer(holder.get(), RELATIONSHIP_TOKENIZED); }
From source file:org.apache.ofbiz.accounting.invoice.InvoiceServices.java
public static Map<String, Object> importInvoice(DispatchContext dctx, Map<String, Object> context) { Locale locale = (Locale) context.get("locale"); Delegator delegator = dctx.getDelegator(); LocalDispatcher dispatcher = dctx.getDispatcher(); GenericValue userLogin = (GenericValue) context.get("userLogin"); ByteBuffer fileBytes = (ByteBuffer) context.get("uploadedFile"); String organizationPartyId = (String) context.get("organizationPartyId"); String encoding = System.getProperty("file.encoding"); String csvString = Charset.forName(encoding).decode(fileBytes).toString(); final BufferedReader csvReader = new BufferedReader(new StringReader(csvString)); CSVFormat fmt = CSVFormat.DEFAULT.withHeader(); List<String> errMsgs = new LinkedList<String>(); List<String> newErrMsgs = new LinkedList<String>(); String lastInvoiceId = null;/*from w ww .j a va2s . com*/ String currentInvoiceId = null; String newInvoiceId = null; int invoicesCreated = 0; if (fileBytes == null) { return ServiceUtil .returnError(UtilProperties.getMessage(resource, "AccountingUploadedFileDataNotFound", locale)); } try { for (final CSVRecord rec : fmt.parse(csvReader)) { currentInvoiceId = rec.get("invoiceId"); if (lastInvoiceId == null || !currentInvoiceId.equals(lastInvoiceId)) { newInvoiceId = null; Map<String, Object> invoice = UtilMisc.toMap("invoiceTypeId", rec.get("invoiceTypeId"), "partyIdFrom", rec.get("partyIdFrom"), "partyId", rec.get("partyId"), "invoiceDate", rec.get("invoiceDate"), "dueDate", rec.get("dueDate"), "currencyUomId", rec.get("currencyUomId"), "description", rec.get("description"), "referenceNumber", rec.get("referenceNumber") + " Imported: orginal InvoiceId: " + currentInvoiceId, "userLogin", userLogin); // replace values if required if (UtilValidate.isNotEmpty(rec.get("partyIdFromTrans"))) { invoice.put("partyIdFrom", rec.get("partyIdFromTrans")); } if (UtilValidate.isNotEmpty(rec.get("partyIdTrans"))) { invoice.put("partyId", rec.get("partyIdTrans")); } // invoice validation try { newErrMsgs = new LinkedList<String>(); if (UtilValidate.isEmpty(invoice.get("partyIdFrom"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory Party Id From and Party Id From Trans missing for invoice: " + currentInvoiceId); } else if (EntityQuery.use(delegator).from("Party") .where("partyId", invoice.get("partyIdFrom")).queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyIdFrom: " + invoice.get("partyIdFrom") + " not found for invoice: " + currentInvoiceId); } if (UtilValidate.isEmpty(invoice.get("partyId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory Party Id and Party Id Trans missing for invoice: " + currentInvoiceId); } else if (EntityQuery.use(delegator).from("Party").where("partyId", invoice.get("partyId")) .queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + invoice.get("partyId") + " not found for invoice: " + currentInvoiceId); } if (UtilValidate.isEmpty(invoice.get("invoiceTypeId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory Invoice Type missing for invoice: " + currentInvoiceId); } else if (EntityQuery.use(delegator).from("InvoiceType") .where("invoiceTypeId", invoice.get("invoiceTypeId")).queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": InvoiceItem type id: " + invoice.get("invoiceTypeId") + " not found for invoice: " + currentInvoiceId); } Boolean isPurchaseInvoice = EntityTypeUtil.hasParentType(delegator, "InvoiceType", "invoiceTypeId", (String) invoice.get("invoiceTypeId"), "parentTypeId", "PURCHASE_INVOICE"); Boolean isSalesInvoice = EntityTypeUtil.hasParentType(delegator, "InvoiceType", "invoiceTypeId", (String) invoice.get("invoiceTypeId"), "parentTypeId", "SALES_INVOICE"); if (isPurchaseInvoice && !invoice.get("partyId").equals(organizationPartyId)) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": A purchase type invoice should have the partyId 'To' being the organizationPartyId(=" + organizationPartyId + ")! however is " + invoice.get("partyId") + "! invoice: " + currentInvoiceId); } if (isSalesInvoice && !invoice.get("partyIdFrom").equals(organizationPartyId)) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": A sales type invoice should have the partyId 'from' being the organizationPartyId(=" + organizationPartyId + ")! however is " + invoice.get("partyIdFrom") + "! invoice: " + currentInvoiceId); } } catch (GenericEntityException e) { Debug.logError("Valication checking problem against database. due to " + e.getMessage(), module); } if (newErrMsgs.size() > 0) { errMsgs.addAll(newErrMsgs); } else { Map<String, Object> invoiceResult = null; try { invoiceResult = dispatcher.runSync("createInvoice", invoice); } catch (GenericServiceException e) { csvReader.close(); Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } newInvoiceId = (String) invoiceResult.get("invoiceId"); invoicesCreated++; } lastInvoiceId = currentInvoiceId; } if (newInvoiceId != null) { Map<String, Object> invoiceItem = UtilMisc.toMap("invoiceId", newInvoiceId, "invoiceItemSeqId", rec.get("invoiceItemSeqId"), "invoiceItemTypeId", rec.get("invoiceItemTypeId"), "productId", rec.get("productId"), "description", rec.get("itemDescription"), "amount", rec.get("amount"), "quantity", rec.get("quantity"), "userLogin", userLogin); if (UtilValidate.isNotEmpty(rec.get("productIdTrans"))) { invoiceItem.put("productId", rec.get("productIdTrans")); } // invoice item validation try { newErrMsgs = new LinkedList<String>(); if (UtilValidate.isEmpty(invoiceItem.get("invoiceItemSeqId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory item sequence Id missing for invoice: " + currentInvoiceId); } if (UtilValidate.isEmpty(invoiceItem.get("invoiceItemTypeId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory invoice item type missing for invoice: " + currentInvoiceId); } else if (EntityQuery.use(delegator).from("InvoiceItemType") .where("invoiceItemTypeId", invoiceItem.get("invoiceItemTypeId")) .queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": InvoiceItem Item type id: " + invoiceItem.get("invoiceItemTypeId") + " not found for invoice: " + currentInvoiceId + " Item seqId:" + invoiceItem.get("invoiceItemSeqId")); } if (UtilValidate.isEmpty(invoiceItem.get("productId")) && UtilValidate.isEmpty(invoiceItem.get("description"))) { } if (UtilValidate.isNotEmpty(invoiceItem.get("productId")) && EntityQuery.use(delegator).from("Product") .where("productId", invoiceItem.get("productId")).queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Product Id: " + invoiceItem.get("productId") + " not found for invoice: " + currentInvoiceId + " Item seqId:" + invoiceItem.get("invoiceItemSeqId")); } if (UtilValidate.isEmpty(invoiceItem.get("amount")) && UtilValidate.isEmpty(invoiceItem.get("quantity"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Either or both quantity and amount is required for invoice: " + currentInvoiceId + " Item seqId:" + invoiceItem.get("invoiceItemSeqId")); } } catch (GenericEntityException e) { Debug.logError("Validation checking problem against database. due to " + e.getMessage(), module); } if (newErrMsgs.size() > 0) { errMsgs.addAll(newErrMsgs); } else { try { dispatcher.runSync("createInvoiceItem", invoiceItem); } catch (GenericServiceException e) { csvReader.close(); Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } } } } } catch (IOException e) { Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } if (errMsgs.size() > 0) { return ServiceUtil.returnError(errMsgs); } Map<String, Object> result = ServiceUtil.returnSuccess(UtilProperties.getMessage(resource, "AccountingNewInvoicesCreated", UtilMisc.toMap("invoicesCreated", invoicesCreated), locale)); result.put("organizationPartyId", organizationPartyId); return result; }
From source file:org.apache.ofbiz.party.party.PartyServices.java
public static Map<String, Object> importParty(DispatchContext dctx, Map<String, Object> context) { Delegator delegator = dctx.getDelegator(); LocalDispatcher dispatcher = dctx.getDispatcher(); Locale locale = (Locale) context.get("locale"); GenericValue userLogin = (GenericValue) context.get("userLogin"); ByteBuffer fileBytes = (ByteBuffer) context.get("uploadedFile"); String encoding = System.getProperty("file.encoding"); String csvString = Charset.forName(encoding).decode(fileBytes).toString(); final BufferedReader csvReader = new BufferedReader(new StringReader(csvString)); CSVFormat fmt = CSVFormat.DEFAULT.withHeader(); List<String> errMsgs = new LinkedList<String>(); List<String> newErrMsgs = new LinkedList<String>(); String lastPartyId = null; // last partyId read from the csv file String currentPartyId = null; // current partyId from the csv file String newPartyId = null; // new to create/update partyId in the system String newCompanyPartyId = null; int partiesCreated = 0; Map<String, Object> result = null; String newContactMechId = null; String currentContactMechTypeId = null; String lastAddress1 = null;//from w w w . j av a2 s . c o m String lastAddress2 = null; String lastCity = null; String lastCountryGeoId = null; String lastEmailAddress = null; String lastCountryCode = null; String lastAreaCode = null; String lastContactNumber = null; String lastContactMechPurposeTypeId = null; String currentContactMechPurposeTypeId = null; Boolean addParty = false; // when modify party, contact mech not added again if (fileBytes == null) { return ServiceUtil .returnError(UtilProperties.getMessage(resourceError, "PartyUploadedFileDataNotFound", locale)); } try { for (final CSVRecord rec : fmt.parse(csvReader)) { if (UtilValidate.isNotEmpty(rec.get("partyId"))) { currentPartyId = rec.get("partyId"); } if (lastPartyId == null || !currentPartyId.equals(lastPartyId)) { newPartyId = null; currentContactMechPurposeTypeId = null; lastAddress1 = null; lastAddress2 = null; lastCity = null; lastCountryGeoId = null; lastEmailAddress = null; lastCountryCode = null; lastAreaCode = null; lastContactNumber = null; // party validation List<GenericValue> currencyCheck = EntityQuery.use(delegator).from("Uom").where("abbreviation", rec.get("preferredCurrencyUomId"), "uomTypeId", "CURRENCY_MEASURE").queryList(); if (UtilValidate.isNotEmpty(rec.get("preferredCurrencyUomId")) && currencyCheck.size() == 0) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Currency code not found for: " + rec.get("preferredCurrencyUomId")); } if (UtilValidate.isEmpty(rec.get("roleTypeId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory roletype is missing, possible values: CUSTOMER, SUPPLIER, EMPLOYEE and more...."); } else if (EntityQuery.use(delegator).from("RoleType") .where("roleTypeId", rec.get("roleTypeId")).queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": RoletypeId is not valid: " + rec.get("roleTypeId")); } if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && EntityQuery.use(delegator) .from("ContactMechType").where("contactMechTypeId", rec.get("contactMechTypeId")) .cache().queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + " contactMechTypeId code not found for: " + rec.get("contactMechTypeId")); } if (UtilValidate.isNotEmpty(rec.get("contactMechPurposeTypeId")) && EntityQuery.use(delegator).from("ContactMechPurposeType") .where("contactMechPurposeTypeId", rec.get("contactMechPurposeTypeId")).cache() .queryOne() == null) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "contactMechPurposeTypeId code not found for: " + rec.get("contactMechPurposeTypeId")); } if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "POSTAL_ADDRESS".equals(rec.get("contactMechTypeId"))) { if (UtilValidate.isEmpty(rec.get("countryGeoId"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Country code missing"); } else { List<GenericValue> countryCheck = EntityQuery.use(delegator).from("Geo") .where("geoTypeId", "COUNTRY", "abbreviation", rec.get("countryGeoId")) .queryList(); if (countryCheck.size() == 0) { newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid Country code: " + rec.get("countryGeoId")); } } if (UtilValidate.isEmpty(rec.get("city"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + "City name is missing"); } if (UtilValidate.isNotEmpty(rec.get("stateProvinceGeoId"))) { List<GenericValue> stateCheck = EntityQuery.use(delegator).from("Geo") .where("geoTypeId", "STATE", "abbreviation", rec.get("stateProvinceGeoId")) .queryList(); if (stateCheck.size() == 0) { newErrMsgs .add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid stateProvinceGeoId code: " + rec.get("countryGeoId")); } } } if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "TELECOM_NUMBER".equals(rec.get("contactMechTypeId"))) { if (UtilValidate.isEmpty(rec.get("telAreaCode")) && UtilValidate.isEmpty(rec.get("telAreaCode"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " telephone number missing"); } } if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "EMAIL_ADDRESS".equals(rec.get("contactMechTypeId"))) { if (UtilValidate.isEmpty(rec.get("emailAddress"))) { newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " email address missing"); } } if (errMsgs.size() == 0) { List<GenericValue> partyCheck = EntityQuery.use(delegator).from("PartyIdentification") .where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")) .queryList(); addParty = partyCheck.size() == 0; if (!addParty) { // update party newPartyId = EntityUtil.getFirst(partyCheck).getString("partyId"); if (UtilValidate.isNotEmpty(rec.get("groupName"))) { Map<String, Object> partyGroup = UtilMisc.toMap("partyId", newPartyId, "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin); result = dispatcher.runSync("updatePartyGroup", partyGroup); } else { // person Map<String, Object> person = UtilMisc.toMap("partyId", newPartyId, "firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "userLogin", userLogin); result = dispatcher.runSync("updatePerson", person); } } else { // create new party if (UtilValidate.isNotEmpty(rec.get("groupName"))) { Map<String, Object> partyGroup = UtilMisc.toMap("preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin, "statusId", "PARTY_ENABLED"); result = dispatcher.runSync("createPartyGroup", partyGroup); } else { // person Map<String, Object> person = UtilMisc.toMap("firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "statusId", "PARTY_ENABLED", "userLogin", userLogin); result = dispatcher.runSync("createPerson", person); } newPartyId = (String) result.get("partyId"); Map<String, Object> partyIdentification = UtilMisc.toMap("partyId", newPartyId, "partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId"), "userLogin", userLogin); result = dispatcher.runSync("createPartyIdentification", partyIdentification); Map<String, Object> partyRole = UtilMisc.toMap("partyId", newPartyId, "roleTypeId", rec.get("roleTypeId"), "userLogin", userLogin); dispatcher.runSync("createPartyRole", partyRole); if (UtilValidate.isNotEmpty(rec.get("companyPartyId"))) { List<GenericValue> companyCheck = EntityQuery.use(delegator) .from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")) .queryList(); if (companyCheck.size() == 0) { // update party group // company does not exist so create Map<String, Object> companyPartyGroup = UtilMisc.toMap("partyId", newCompanyPartyId, "statusId", "PARTY_ENABLED", "userLogin", userLogin); result = dispatcher.runSync("createPartyGroup", companyPartyGroup); newCompanyPartyId = (String) result.get("partyId"); } else { newCompanyPartyId = EntityUtil.getFirst(companyCheck).getString("partyId"); } Map<String, Object> companyRole = UtilMisc.toMap("partyId", newCompanyPartyId, "roleTypeId", "ACCOUNT", "userLogin", userLogin); dispatcher.runSync("createPartyRole", companyRole); // company exist, so create link Map<String, Object> partyRelationship = UtilMisc.toMap("partyIdTo", newPartyId, "partyIdFrom", newCompanyPartyId, "roleTypeIdFrom", "ACCOUNT", "partyRelationshipTypeId", "EMPLOYMENT", "userLogin", userLogin); result = dispatcher.runSync("createPartyRelationship", partyRelationship); } } Debug.logInfo(" =========================================================party created id: " + newPartyId, module); partiesCreated++; } else { errMsgs.addAll(newErrMsgs); newErrMsgs = new LinkedList<String>(); } } currentContactMechTypeId = rec.get("contactMechTypeId"); currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId"); // party correctly created (not updated) and contactMechtype provided? if (newPartyId != null && addParty && UtilValidate.isNotEmpty(currentContactMechTypeId)) { // fill maps and check changes Map<String, Object> emailAddress = UtilMisc.toMap("contactMechTypeId", "EMAIL_ADDRESS", "userLogin", userLogin); Boolean emailAddressChanged = false; if ("EMAIL_ADDRESS".equals(currentContactMechTypeId)) { emailAddress.put("infoString", rec.get("emailAddress")); emailAddressChanged = lastEmailAddress == null || !lastEmailAddress.equals(rec.get("emailAddress")); lastEmailAddress = rec.get("emailAddress"); } Map<String, Object> postalAddress = UtilMisc.toMap("userLogin", (Object) userLogin); // casting is here necessary for some compiler versions Boolean postalAddressChanged = false; if ("POSTAL_ADDRESS".equals(currentContactMechTypeId)) { postalAddress.put("address1", rec.get("address1")); postalAddress.put("address2", rec.get("address2")); postalAddress.put("city", rec.get("city")); postalAddress.put("stateProvinceGeoId", rec.get("stateProvinceGeoId")); postalAddress.put("countryGeoId", rec.get("countryGeoId")); postalAddress.put("postalCode", rec.get("postalCode")); postalAddressChanged = lastAddress1 == null || !lastAddress1.equals(postalAddress.get("address1")) || lastAddress2 == null || !lastAddress2.equals(postalAddress.get("address2")) || lastCity == null || !lastCity.equals(postalAddress.get("city")) || lastCountryGeoId == null || !lastCountryGeoId.equals(postalAddress.get("countryGeoId")); lastAddress1 = (String) postalAddress.get("address1"); lastAddress2 = (String) postalAddress.get("address2"); lastCity = (String) postalAddress.get("city"); lastCountryGeoId = (String) postalAddress.get("countryGeoId"); } Map<String, Object> telecomNumber = UtilMisc.toMap("userLogin", (Object) userLogin); // casting is here necessary for some compiler versions Boolean telecomNumberChanged = false; if ("TELECOM_NUMBER".equals(currentContactMechTypeId)) { telecomNumber.put("countryCode", rec.get("telCountryCode")); telecomNumber.put("areaCode", rec.get("telAreaCode")); telecomNumber.put("contactNumber", rec.get("telContactNumber")); telecomNumberChanged = lastCountryCode == null || !lastCountryCode.equals(telecomNumber.get("countryCode")) || lastAreaCode == null || !lastAreaCode.equals(telecomNumber.get("areaCode")) || lastContactNumber == null || !lastContactNumber.equals(telecomNumber.get("contactNumber")); lastCountryCode = (String) telecomNumber.get("countryCode"); lastAreaCode = (String) telecomNumber.get("areaCode"); lastContactNumber = (String) telecomNumber.get("contactNumber"); } Map<String, Object> partyContactMechPurpose = UtilMisc.toMap("partyId", newPartyId, "userLogin", userLogin); Boolean partyContactMechPurposeChanged = false; currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId"); if (currentContactMechPurposeTypeId != null && ("TELECOM_NUMBER".equals(currentContactMechTypeId) || "POSTAL_ADDRESS".equals(currentContactMechTypeId) || "EMAIL_ADDRESS".equals(currentContactMechTypeId))) { partyContactMechPurpose.put("contactMechPurposeTypeId", currentContactMechPurposeTypeId); partyContactMechPurposeChanged = (lastContactMechPurposeTypeId == null || !lastContactMechPurposeTypeId.equals(currentContactMechPurposeTypeId)) && !telecomNumberChanged && !postalAddressChanged && !emailAddressChanged; Debug.logInfo( "===================================last:" + lastContactMechPurposeTypeId + " current: " + currentContactMechPurposeTypeId + " t :" + telecomNumberChanged + " p: " + postalAddressChanged + " e: " + emailAddressChanged + " result: " + partyContactMechPurposeChanged, module); } lastContactMechPurposeTypeId = currentContactMechPurposeTypeId; // update if (errMsgs.size() == 0) { if (postalAddressChanged) { result = dispatcher.runSync("createPostalAddress", postalAddress); newContactMechId = (String) result.get("contactMechId"); if (currentContactMechPurposeTypeId == null) { currentContactMechPurposeTypeId = "GENERAL_LOCATION"; } dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin)); } if (telecomNumberChanged) { result = dispatcher.runSync("createTelecomNumber", telecomNumber); newContactMechId = (String) result.get("contactMechId"); if (currentContactMechPurposeTypeId == null) { currentContactMechPurposeTypeId = "PHONE_WORK"; } dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin)); } if (emailAddressChanged) { result = dispatcher.runSync("createContactMech", emailAddress); newContactMechId = (String) result.get("contactMechId"); if (currentContactMechPurposeTypeId == null) { currentContactMechPurposeTypeId = "PRIMARY_EMAIL"; } dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin)); } if (partyContactMechPurposeChanged) { partyContactMechPurpose.put("contactMechId", newContactMechId); result = dispatcher.runSync("createPartyContactMechPurpose", partyContactMechPurpose); } lastPartyId = currentPartyId; errMsgs.addAll(newErrMsgs); newErrMsgs = new LinkedList<String>(); } } } } catch (GenericServiceException e) { Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } catch (GenericEntityException e) { Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } catch (IOException e) { Debug.logError(e, module); return ServiceUtil.returnError(e.getMessage()); } if (errMsgs.size() > 0) { return ServiceUtil.returnError(errMsgs); } result = ServiceUtil.returnSuccess(UtilProperties.getMessage(resource, "PartyNewPartiesCreated", UtilMisc.toMap("partiesCreated", partiesCreated), locale)); return result; }
From source file:org.apache.phoenix.mapreduce.CsvToKeyValueMapperTest.java
@Test public void testCsvLineParser() throws IOException { CsvToKeyValueMapper.CsvLineParser lineParser = new CsvToKeyValueMapper.CsvLineParser(';', '"', '\\'); CSVRecord parsed = lineParser.parse("one;two"); assertEquals("one", parsed.get(0)); assertEquals("two", parsed.get(1)); assertTrue(parsed.isConsistent());/* www .ja v a2 s . c o m*/ assertEquals(1, parsed.getRecordNumber()); }
From source file:org.apache.phoenix.mapreduce.CsvToKeyValueMapperTest.java
@Test public void testCsvLineParserWithQuoting() throws IOException { CsvToKeyValueMapper.CsvLineParser lineParser = new CsvToKeyValueMapper.CsvLineParser(';', '"', '\\'); CSVRecord parsed = lineParser.parse("\"\\\"one\";\"\\;two\\\\\""); assertEquals("\"one", parsed.get(0)); assertEquals(";two\\", parsed.get(1)); assertTrue(parsed.isConsistent());/* w ww. ja v a 2 s. co m*/ assertEquals(1, parsed.getRecordNumber()); }
From source file:org.apache.phoenix.pherf.result.impl.CSVFileResultHandler.java
public synchronized List<Result> read() throws IOException { CSVParser parser = null;/*from ww w . jav a 2 s . com*/ util.ensureBaseResultDirExists(); try { File file = new File(resultFileName); parser = CSVParser.parse(file, Charset.defaultCharset(), CSVFormat.DEFAULT); List<CSVRecord> records = parser.getRecords(); List<Result> results = new ArrayList<>(); String header = null; for (CSVRecord record : records) { // First record is the CSV Header if (record.getRecordNumber() == 1) { header = record.toString(); continue; } List<ResultValue> resultValues = new ArrayList<>(); for (String val : record.toString().split(PherfConstants.RESULT_FILE_DELIMETER)) { resultValues.add(new ResultValue(val)); } Result result = new Result(resultFileDetails, header, resultValues); results.add(result); } return results; } finally { parser.close(); } }
From source file:org.etudes.mneme.tool.ImportOfflineView.java
/** * {@inheritDoc}//from w w w. j ava 2 s . co m */ public void post(HttpServletRequest req, HttpServletResponse res, Context context, String[] params) throws IOException { String returnUrl = (params.length > 3) ? params[2] : ""; String sort = (params.length > 3) ? params[3] : "0A"; String siteId = toolManager.getCurrentPlacement().getContext(); if (!this.assessmentService.allowManageAssessments(siteId)) { // redirect to error res.sendRedirect(res.encodeRedirectURL(Web.returnUrl(req, "/error/" + Errors.unauthorized))); return; } Site site = null; try { site = this.siteService.getSite(siteId); } catch (IdUnusedException e) { // redirect to error res.sendRedirect(res.encodeRedirectURL(Web.returnUrl(req, "/error/" + Errors.unauthorized))); return; } String iidCode = this.rosterService.findInstitutionCode(site.getTitle()); // a CSV uploader for the CSV file UploadCsv upload = new UploadCsv(); context.put("upload", upload); // read the form String destination = uiService.decode(req, context); // import the offlines if ("UPLOAD".equals(destination)) { List<GradeImportSet> importSets = new ArrayList<GradeImportSet>(1); // get the Tool session, where we will be storing this until confirmed ToolSession toolSession = sessionManager.getCurrentToolSession(); toolSession.setAttribute(GradeImportSet.ATTR_NAME, importSets); // col 0, the student id col 1 .. n, the title of the offline assessment, in the first record, and the score, in the rest List<CSVRecord> records = upload.getRecords(); if ((records != null) && (records.size() > 1)) { // the header record CSVRecord header = records.get(0); // find the "Student ID" column int idCol = -1; for (int c = 0; c < header.size(); c++) { // get the assessment title, extras String title = StringUtil.trimToZero(header.get(c)); if ("Student ID".equalsIgnoreCase(title)) { idCol = c; break; } } if (idCol != -1) { // try all other columns for (int i = 0; i < header.size(); i++) { // we use this column for the Student ID if (i == idCol) continue; // we will keep this ONLY if we see at least one valid grade entry GradeImportSet set = new GradeImportSet(); boolean hasValidEntries = false; // get the assessment title, points String title = StringUtil.trimToZero(header.get(i)); Float points = null; int extraPos = title.indexOf("{{"); if (extraPos != -1) { String extra = StringUtil.trimToZero(title.substring(extraPos + 2, title.length() - 2)); title = StringUtil.trimToZero(title.substring(0, extraPos)); try { points = Float.valueOf(extra); } catch (NumberFormatException e) { } } // skip blank header columns if (title.length() == 0) continue; // these two titles we also skip, as created by UCB Gradebook's export if ("Section ID".equals(title)) continue; if ("Cumulative".equals(title)) continue; // new assessment or existing Assessment existing = assessmentService.assessmentExists(siteId, title); if (existing != null) { // just ignore if it does not take points if (!existing.getHasPoints()) { continue; } set.assessment = existing; } else { set.assessmentTitle = title; set.points = points; } for (CSVRecord r : records) { // skip the header if (r.getRecordNumber() == 1) continue; GradeImport x = new GradeImport(); set.rows.add(x); if (r.size() >= 1) { x.studentId = r.get(idCol); } if (r.size() > i) { x.scoreGiven = r.get(i); } if (x.scoreGiven != null) { try { x.score = Float.parseFloat(x.scoreGiven); } catch (NumberFormatException e) { } } if ((x.score != null) & (x.studentId != null)) { // use value in paren, if there String id = x.studentId; int parenPos = id.indexOf("("); if (parenPos != -1) { id = StringUtil.trimToZero(id.substring(parenPos + 1, id.length() - 1)); } User user = findIdentifiedStudentInSite(id, siteId, iidCode); if (user != null) { // check for duplicate records boolean duplicate = false; for (GradeImport gi : set.rows) { if (gi == x) continue; if (gi.userId == null) continue; if (gi.userId.equals(user.getId())) { duplicate = true; x.duplicate = Boolean.TRUE; break; } } if (!duplicate) { x.userId = user.getId(); x.name = user.getDisplayName(); hasValidEntries = true; } } } } // keep this ONLY if we see at least one valid grade entry if (hasValidEntries) { importSets.add(set); } } } } destination = "/confirm_grades_import/" + returnUrl + "/" + sort; } res.sendRedirect(res.encodeRedirectURL(Web.returnUrl(req, destination))); }