List of usage examples for org.apache.commons.csv CSVParser close
@Override public void close() throws IOException
From source file:canreg.client.dataentry.Convert.java
public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { boolean success = false; Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>(); noNeedToLookAtPatientVariables/*from w w w. j a v a 2 s . c o m*/ .add(canreg.common.Tools.toLowerCaseStandardized(io.getPatientIDVariableName())); noNeedToLookAtPatientVariables .add(canreg.common.Tools.toLowerCaseStandardized(io.getPatientRecordIDVariableName())); String firstNameVariableName = io.getFirstNameVariableName(); String sexVariableName = io.getSexVariableName(); CSVParser parser = null; HashMap mpCodes = new HashMap(); int numberOfLinesRead = 0; Map<String, Integer> nameSexTable = server.getNameSexTables(); try { // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}"); int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath()); debugOut("Importing data from " + file); CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator()); parser = CSVParser.parse(file, io.getFileCharset(), format); int linesToRead = io.getMaxLines(); if (linesToRead == -1 || linesToRead > numberOfRecordsInFile) { linesToRead = numberOfRecordsInFile; } for (CSVRecord csvRecord : parser) { numberOfLinesRead++; // We allow for null tasks... boolean needToSavePatientAgain = true; int patientDatabaseRecordID = -1; if (task != null) { if (canreg.client.gui.management.CanReg4MigrationInternalFrame.isPaused) { task.firePropertyChange("paused", false, true); } if (!canreg.client.gui.management.CanReg4MigrationInternalFrame.isPaused) { task.firePropertyChange("paused", true, false); task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead, (numberOfLinesRead) * 100 / linesToRead); } } // Build patient part Patient patient = new Patient(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("patient")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { patient.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { patient.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with patient part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // debugOut(patient.toString()); // Build tumour part Tumour tumour = new Tumour(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("tumour") && rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { tumour.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { tumour.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with tumour part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // Build source part Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>()); Source source = new Source(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME) && rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { source.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { source.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with source part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } sources.add(source); tumour.setSources(sources); // debugOut(tumour.toString()); // add patient to the database Object patientID = patient.getVariable(io.getPatientIDVariableName()); Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); if (patientID == null) { // save the record to get the new patientID; patientDatabaseRecordID = server.savePatient(patient); patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME, false); patientID = patient.getVariable(io.getPatientIDVariableName()); patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); } if (io.isDataFromPreviousCanReg()) { // set update date for the patient the same as for the tumour Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName()); patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate); // Set the patientID the same as the tumourID initially // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName()); Object tumourSequence = "1"; String tumourSequenceString = tumourSequence + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; // If this is a multiple primary tumour... String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName()); if (mpCodeString != null && mpCodeString.length() > 0) { patientID = lookUpPatientID(mpCodeString, patientID, mpCodes); // rebuild sequenceNumber Tumour[] tumours = new Tumour[0]; try { tumours = CanRegClientApp.getApplication() .getTumourRecordsBasedOnPatientID(patientID + "", false); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } tumourSequenceString = (tumours.length + 1) + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; Patient[] oldPatients = null; try { oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID, false); } catch (RemoteException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SecurityException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (RecordLockedException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } for (Patient oldPatient : oldPatients) { if (!Tools.newRecordContainsNewInfo(patient, oldPatient, noNeedToLookAtPatientVariables)) { needToSavePatientAgain = false; patient = oldPatient; patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName()); } } } Object tumourID = patientRecordID + "" + tumourSequenceString; // patient.setVariable(io.getPatientIDVariableName(), patientID); tumour.setVariable(io.getTumourIDVariablename(), tumourID); // And store the record ID patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID); // Set the patient ID number on the tumour tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); // Set the deprecated flag to 0 - no obsolete records from CR4 tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0"); patient.setVariable(io.getObsoletePatientFlagVariableName(), "0"); } // Set the name in the firstName database String sex = (String) patient.getVariable(sexVariableName); if (sex != null && sex.length() > 0) { Integer sexCode = Integer.parseInt(sex); String firstNames = (String) patient.getVariable(firstNameVariableName); if (firstNames != null) { String[] firstNamesArray = firstNames.split(" "); for (String firstName : firstNamesArray) { if (firstName != null && firstName.trim().length() > 0) { // here we use the locale specific toUpperCase Integer registeredSexCode = nameSexTable.get(firstName); if (registeredSexCode == null) { NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, false); nameSexTable.put(firstName, sexCode); } else if (registeredSexCode != sexCode) { if (registeredSexCode != 9) { sexCode = 9; NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, true); nameSexTable.remove(firstName); nameSexTable.put(firstName, sexCode); } } } } } } if (needToSavePatientAgain) { if (patientDatabaseRecordID > 0) { server.editPatient(patient); } else { patientDatabaseRecordID = server.savePatient(patient); } } if (patient != null && tumour != null) { String icd10 = (String) tumour.getVariable(io.getICD10VariableName()); if (icd10 == null || icd10.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour); tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue()); } } if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); } if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); } int tumourDatabaseIDNumber = server.saveTumour(tumour); if (Thread.interrupted()) { //We've been interrupted: no more importing. throw new InterruptedException(); } } task.firePropertyChange("finished", null, null); success = true; } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (InterruptedException ex) { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex); success = true; } catch (IndexOutOfBoundsException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } finally { if (parser != null) { try { parser.close(); } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } } } return success; }
From source file:canreg.client.dataentry.Import.java
/** * * @param task//from w w w . ja va 2 s. c o m * @param doc * @param map * @param file * @param server * @param io * @return * @throws java.sql.SQLException * @throws java.rmi.RemoteException * @throws canreg.server.database.RecordLockedException */ public static boolean importFile(Task<Object, String> task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { //public static boolean importFile(canreg.client.gui.management.CanReg4MigrationInternalFrame.MigrationTask task, Document doc, List<canreg.client.dataentry.Relation> map, File file, CanRegServerInterface server, ImportOptions io) throws SQLException, RemoteException, SecurityException, RecordLockedException { boolean success = false; Set<String> noNeedToLookAtPatientVariables = new TreeSet<String>(); noNeedToLookAtPatientVariables.add(io.getPatientIDVariableName()); noNeedToLookAtPatientVariables.add(io.getPatientRecordIDVariableName()); String firstNameVariableName = io.getFirstNameVariableName(); String sexVariableName = io.getSexVariableName(); CSVParser parser = null; CSVFormat format = CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(io.getSeparator()); int linesToRead = io.getMaxLines(); HashMap mpCodes = new HashMap(); int numberOfLinesRead = 0; Map<String, Integer> nameSexTable = server.getNameSexTables(); try { // FileInputStream fis = new FileInputStream(file); // BufferedReader bsr = new BufferedReader(new InputStreamReader(fis, io.getFileCharset())); // Logger.getLogger(Import.class.getName()).log(Level.CONFIG, "Name of the character encoding {0}"); int numberOfRecordsInFile = canreg.common.Tools.numberOfLinesInFile(file.getAbsolutePath()); if (linesToRead > 0) { linesToRead = Math.min(numberOfRecordsInFile, linesToRead); } else { linesToRead = numberOfRecordsInFile; } parser = CSVParser.parse(file, io.getFileCharset(), format); for (CSVRecord csvRecord : parser) { numberOfLinesRead++; // We allow for null tasks... boolean needToSavePatientAgain = true; int patientDatabaseRecordID = -1; if (task != null) { task.firePropertyChange("progress", (numberOfLinesRead - 1) * 100 / linesToRead, (numberOfLinesRead) * 100 / linesToRead); } // Build patient part Patient patient = new Patient(); for (int i = 0; i < map.size(); i++) { Relation rel = map.get(i); if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("patient")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { patient.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { patient.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with patient part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // debugOut(patient.toString()); // Build tumour part Tumour tumour = new Tumour(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase("tumour")) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { tumour.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { tumour.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with tumour part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } // Build source part Set<Source> sources = Collections.synchronizedSet(new LinkedHashSet<Source>()); Source source = new Source(); for (canreg.client.dataentry.Relation rel : map) { if (rel.getDatabaseTableVariableID() >= 0 && rel.getDatabaseTableName().equalsIgnoreCase(Globals.SOURCE_TABLE_NAME)) { if (rel.getFileColumnNumber() < csvRecord.size()) { if (rel.getVariableType().equalsIgnoreCase("Number")) { if (csvRecord.get(rel.getFileColumnNumber()).length() > 0) { try { source.setVariable(rel.getDatabaseVariableName(), Integer.parseInt(csvRecord.get(rel.getFileColumnNumber()))); } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Number format error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } } } else { source.setVariable(rel.getDatabaseVariableName(), StringEscapeUtils.unescapeCsv(csvRecord.get(rel.getFileColumnNumber()))); } } else { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Something wrong with source part of line " + numberOfLinesRead + ".", new Exception("Error in line: " + numberOfLinesRead + ". Can't find field: " + rel.getDatabaseVariableName())); } } } sources.add(source); tumour.setSources(sources); // debugOut(tumour.toString()); // add patient to the database Object patientID = patient.getVariable(io.getPatientIDVariableName()); Object patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); if (patientID == null) { // save the record to get the new patientID; patientDatabaseRecordID = server.savePatient(patient); patient = (Patient) server.getRecord(patientDatabaseRecordID, Globals.PATIENT_TABLE_NAME, false); patientID = patient.getVariable(io.getPatientIDVariableName()); patientRecordID = patient.getVariable(io.getPatientRecordIDVariableName()); } if (io.isDataFromPreviousCanReg()) { // set update date for the patient the same as for the tumour Object updateDate = tumour.getVariable(io.getTumourUpdateDateVariableName()); patient.setVariable(io.getPatientUpdateDateVariableName(), updateDate); // Set the patientID the same as the tumourID initially // Object tumourSequence = tumour.getVariable(io.getTumourSequenceVariableName()); Object tumourSequence = "1"; String tumourSequenceString = tumourSequence + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; // If this is a multiple primary tumour... String mpCodeString = (String) tumour.getVariable(io.getMultiplePrimaryVariableName()); if (mpCodeString != null && mpCodeString.length() > 0) { patientID = lookUpPatientID(mpCodeString, patientID, mpCodes); // rebuild sequenceNumber Tumour[] tumours = new Tumour[0]; try { tumours = CanRegClientApp.getApplication() .getTumourRecordsBasedOnPatientID(patientID + "", false); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } tumourSequenceString = (tumours.length + 1) + ""; while (tumourSequenceString.length() < Globals.ADDITIONAL_DIGITS_FOR_PATIENT_RECORD) { tumourSequenceString = "0" + tumourSequenceString; } patientRecordID = patientID + "" + tumourSequenceString; Patient[] oldPatients = null; try { oldPatients = CanRegClientApp.getApplication().getPatientRecordsByID((String) patientID, false); } catch (RemoteException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SecurityException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (DistributedTableDescriptionException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (RecordLockedException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } catch (UnknownTableException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } for (Patient oldPatient : oldPatients) { if (!Tools.newRecordContainsNewInfo(patient, oldPatient, noNeedToLookAtPatientVariables)) { needToSavePatientAgain = false; patient = oldPatient; patientRecordID = oldPatient.getVariable(io.getPatientRecordIDVariableName()); } } } Object tumourID = patientRecordID + "" + tumourSequenceString; // patient.setVariable(io.getPatientIDVariableName(), patientID); tumour.setVariable(io.getTumourIDVariablename(), tumourID); // And store the record ID patient.setVariable(io.getPatientRecordIDVariableName(), patientRecordID); // Set the patient ID number on the tumour tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); // Set the deprecated flag to 0 - no obsolete records from CR4 tumour.setVariable(io.getObsoleteTumourFlagVariableName(), "0"); patient.setVariable(io.getObsoletePatientFlagVariableName(), "0"); } // Set the name in the firstName database String sex = (String) patient.getVariable(sexVariableName); if (sex != null && sex.length() > 0) { Integer sexCode = Integer.parseInt(sex); String firstNames = (String) patient.getVariable(firstNameVariableName); if (firstNames != null) { String[] firstNamesArray = firstNames.split(" "); for (String firstName : firstNamesArray) { if (firstName != null && firstName.trim().length() > 0) { // here we use the locale specific toUpperCase Integer registeredSexCode = nameSexTable.get(firstName); if (registeredSexCode == null) { NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, false); nameSexTable.put(firstName, sexCode); } else if (registeredSexCode != sexCode) { if (registeredSexCode != 9) { sexCode = 9; NameSexRecord nsr = new NameSexRecord(); nsr.setName(firstName); nsr.setSex(sexCode); server.saveNameSexRecord(nsr, true); nameSexTable.remove(firstName); nameSexTable.put(firstName, sexCode); } } } } } } if (needToSavePatientAgain) { if (patientDatabaseRecordID > 0) { server.editPatient(patient); } else { patientDatabaseRecordID = server.savePatient(patient); } } if (patient != null && tumour != null) { String icd10 = (String) tumour.getVariable(io.getICD10VariableName()); if (icd10 == null || icd10.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICD10, patient, tumour); tumour.setVariable(io.getICD10VariableName(), conversionResult[0].getValue()); } String iccc = (String) tumour.getVariable(io.getICCCVariableName()); if (iccc == null || iccc.trim().length() == 0) { ConversionResult[] conversionResult = canreg.client.CanRegClientApp.getApplication() .performConversions(Converter.ConversionName.ICDO3toICCC3, patient, tumour); tumour.setVariable(io.getICCCVariableName(), conversionResult[0].getValue()); } } if (tumour.getVariable(io.getPatientIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientIDTumourTableVariableName(), patientID); } if (tumour.getVariable(io.getPatientRecordIDTumourTableVariableName()) == null) { tumour.setVariable(io.getPatientRecordIDTumourTableVariableName(), patientRecordID); } int tumourDatabaseIDNumber = server.saveTumour(tumour); if (Thread.interrupted()) { //We've been interrupted: no more importing. throw new InterruptedException(); } } task.firePropertyChange("finished", null, null); success = true; } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (NumberFormatException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (InterruptedException ex) { Logger.getLogger(Import.class.getName()).log(Level.INFO, "Interupted on line: " + (numberOfLinesRead + 1) + ". ", ex); success = true; } catch (IndexOutOfBoundsException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } catch (SQLException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, "Error in line: " + (numberOfLinesRead + 1 + 1) + ". ", ex); success = false; } finally { if (parser != null) { try { parser.close(); } catch (IOException ex) { Logger.getLogger(Import.class.getName()).log(Level.SEVERE, null, ex); } } } return success; }
From source file:edu.harvard.mcz.imagecapture.loader.JobVerbatimFieldLoad.java
@Override public void start() { startDateTime = new Date(); Singleton.getSingletonInstance().getJobList().addJob((RunnableJob) this); runStatus = RunStatus.STATUS_RUNNING; String selectedFilename = ""; if (file == null) { final JFileChooser fileChooser = new JFileChooser(); fileChooser.setFileSelectionMode(JFileChooser.FILES_AND_DIRECTORIES); if (Singleton.getSingletonInstance().getProperties().getProperties() .getProperty(ImageCaptureProperties.KEY_LASTLOADPATH) != null) { fileChooser.setCurrentDirectory(new File(Singleton.getSingletonInstance().getProperties() .getProperties().getProperty(ImageCaptureProperties.KEY_LASTLOADPATH))); }// ww w . j a v a 2 s. com int returnValue = fileChooser.showOpenDialog(Singleton.getSingletonInstance().getMainFrame()); if (returnValue == JFileChooser.APPROVE_OPTION) { file = fileChooser.getSelectedFile(); } } if (file != null) { log.debug("Selected file to load: " + file.getName() + "."); if (file.exists() && file.isFile() && file.canRead()) { // Save location Singleton.getSingletonInstance().getProperties().getProperties() .setProperty(ImageCaptureProperties.KEY_LASTLOADPATH, file.getPath()); selectedFilename = file.getName(); String[] headers = new String[] {}; CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader(headers); int rows = 0; try { rows = readRows(file, csvFormat); } catch (FileNotFoundException e) { JOptionPane.showMessageDialog(Singleton.getSingletonInstance().getMainFrame(), "Unable to load data, file not found: " + e.getMessage(), "Error: File Not Found", JOptionPane.OK_OPTION); errors.append("File not found ").append(e.getMessage()).append("\n"); log.error(e.getMessage(), e); } catch (IOException e) { errors.append("Error loading csv format, trying tab delimited: ").append(e.getMessage()) .append("\n"); log.debug(e.getMessage()); try { // try reading as tab delimited format, if successful, use that format. CSVFormat tabFormat = CSVFormat.newFormat('\t').withIgnoreSurroundingSpaces(true) .withHeader(headers).withQuote('"'); rows = readRows(file, tabFormat); csvFormat = tabFormat; } catch (IOException e1) { errors.append("Error Loading data: ").append(e1.getMessage()).append("\n"); log.error(e.getMessage(), e1); } } try { Reader reader = new FileReader(file); CSVParser csvParser = new CSVParser(reader, csvFormat); Map<String, Integer> csvHeader = csvParser.getHeaderMap(); headers = new String[csvHeader.size()]; int i = 0; for (String header : csvHeader.keySet()) { headers[i++] = header; log.debug(header); } boolean okToRun = true; //TODO: Work picking/checking responsibility into a FieldLoaderWizard List<String> headerList = Arrays.asList(headers); if (!headerList.contains("barcode")) { log.error("Input file " + file.getName() + " header does not contain required field 'barcode'."); // no barcode field, we can't match the input to specimen records. errors.append("Field \"barcode\" not found in csv file headers. Unable to load data.") .append("\n"); okToRun = false; } if (okToRun) { Iterator<CSVRecord> iterator = csvParser.iterator(); FieldLoader fl = new FieldLoader(); if (headerList.size() == 3 && headerList.contains("verbatimUnclassifiedText") && headerList.contains("questions") && headerList.contains("barcode")) { log.debug("Input file matches case 1: Unclassified text only."); // Allowed case 1a: unclassified text only int confirm = JOptionPane.showConfirmDialog( Singleton.getSingletonInstance().getMainFrame(), "Confirm load from file " + selectedFilename + " (" + rows + " rows) with just barcode and verbatimUnclassifiedText", "Verbatim unclassified Field found for load", JOptionPane.OK_CANCEL_OPTION); if (confirm == JOptionPane.OK_OPTION) { String barcode = ""; int lineNumber = 0; while (iterator.hasNext()) { lineNumber++; counter.incrementSpecimens(); CSVRecord record = iterator.next(); try { String verbatimUnclassifiedText = record.get("verbatimUnclassifiedText"); barcode = record.get("barcode"); String questions = record.get("questions"); fl.load(barcode, verbatimUnclassifiedText, questions, true); counter.incrementSpecimensUpdated(); } catch (IllegalArgumentException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } catch (LoadException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } percentComplete = (int) ((lineNumber * 100f) / rows); this.setPercentComplete(percentComplete); } } else { errors.append("Load canceled by user.").append("\n"); } } else if (headerList.size() == 4 && headerList.contains("verbatimUnclassifiedText") && headerList.contains("questions") && headerList.contains("barcode") && headerList.contains("verbatimClusterIdentifier")) { log.debug( "Input file matches case 1: Unclassified text only (with cluster identifier)."); // Allowed case 1b: unclassified text only (including cluster identifier) int confirm = JOptionPane.showConfirmDialog( Singleton.getSingletonInstance().getMainFrame(), "Confirm load from file " + selectedFilename + " (" + rows + " rows) with just barcode and verbatimUnclassifiedText", "Verbatim unclassified Field found for load", JOptionPane.OK_CANCEL_OPTION); if (confirm == JOptionPane.OK_OPTION) { String barcode = ""; int lineNumber = 0; while (iterator.hasNext()) { lineNumber++; counter.incrementSpecimens(); CSVRecord record = iterator.next(); try { String verbatimUnclassifiedText = record.get("verbatimUnclassifiedText"); String verbatimClusterIdentifier = record.get("verbatimClusterIdentifier"); barcode = record.get("barcode"); String questions = record.get("questions"); fl.load(barcode, verbatimUnclassifiedText, verbatimClusterIdentifier, questions, true); counter.incrementSpecimensUpdated(); } catch (IllegalArgumentException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } catch (LoadException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } percentComplete = (int) ((lineNumber * 100f) / rows); this.setPercentComplete(percentComplete); } } else { errors.append("Load canceled by user.").append("\n"); } } else if (headerList.size() == 8 && headerList.contains("verbatimUnclassifiedText") && headerList.contains("questions") && headerList.contains("barcode") && headerList.contains("verbatimLocality") && headerList.contains("verbatimDate") && headerList.contains("verbatimNumbers") && headerList.contains("verbatimCollector") && headerList.contains("verbatimCollection")) { // Allowed case two, transcription into verbatim fields, must be exact list of all // verbatim fields, not including cluster identifier or other metadata. log.debug("Input file matches case 2: Full list of verbatim fields."); int confirm = JOptionPane.showConfirmDialog( Singleton.getSingletonInstance().getMainFrame(), "Confirm load from file " + selectedFilename + " (" + rows + " rows) with just barcode and verbatim fields.", "Verbatim Fields found for load", JOptionPane.OK_CANCEL_OPTION); if (confirm == JOptionPane.OK_OPTION) { String barcode = ""; int lineNumber = 0; while (iterator.hasNext()) { lineNumber++; counter.incrementSpecimens(); CSVRecord record = iterator.next(); try { String verbatimLocality = record.get("verbatimLocality"); String verbatimDate = record.get("verbatimDate"); String verbatimCollector = record.get("verbatimCollector"); String verbatimCollection = record.get("verbatimCollection"); String verbatimNumbers = record.get("verbatimNumbers"); String verbatimUnclasifiedText = record.get("verbatimUnclassifiedText"); barcode = record.get("barcode"); String questions = record.get("questions"); fl.load(barcode, verbatimLocality, verbatimDate, verbatimCollector, verbatimCollection, verbatimNumbers, verbatimUnclasifiedText, questions); counter.incrementSpecimensUpdated(); } catch (IllegalArgumentException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } catch (LoadException e) { RunnableJobError error = new RunnableJobError(file.getName(), barcode, Integer.toString(lineNumber), e.getClass().getSimpleName(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(error); log.error(e.getMessage(), e); } percentComplete = (int) ((lineNumber * 100f) / rows); this.setPercentComplete(percentComplete); } } else { errors.append("Load canceled by user.").append("\n"); } } else { // allowed case three, transcription into arbitrary sets verbatim or other fields log.debug("Input file case 3: Arbitrary set of fields."); // Check column headers before starting run. boolean headersOK = false; try { HeaderCheckResult headerCheck = fl.checkHeaderList(headerList); if (headerCheck.isResult()) { int confirm = JOptionPane.showConfirmDialog( Singleton.getSingletonInstance().getMainFrame(), "Confirm load from file " + selectedFilename + " (" + rows + " rows) with headers: \n" + headerCheck.getMessage().replaceAll(":", ":\n"), "Fields found for load", JOptionPane.OK_CANCEL_OPTION); if (confirm == JOptionPane.OK_OPTION) { headersOK = true; } else { errors.append("Load canceled by user.").append("\n"); } } else { int confirm = JOptionPane.showConfirmDialog( Singleton.getSingletonInstance().getMainFrame(), "Problem found with headers in file, try to load anyway?\nHeaders: \n" + headerCheck.getMessage().replaceAll(":", ":\n"), "Problem in fields for load", JOptionPane.OK_CANCEL_OPTION); if (confirm == JOptionPane.OK_OPTION) { headersOK = true; } else { errors.append("Load canceled by user.").append("\n"); } } } catch (LoadException e) { errors.append("Error loading data: \n").append(e.getMessage()).append("\n"); JOptionPane.showMessageDialog(Singleton.getSingletonInstance().getMainFrame(), e.getMessage().replaceAll(":", ":\n"), "Error Loading Data: Problem Fields", JOptionPane.ERROR_MESSAGE); log.error(e.getMessage(), e); } if (headersOK) { int lineNumber = 0; while (iterator.hasNext()) { lineNumber++; Map<String, String> data = new HashMap<String, String>(); CSVRecord record = iterator.next(); String barcode = record.get("barcode"); Iterator<String> hi = headerList.iterator(); boolean containsNonVerbatim = false; while (hi.hasNext()) { String header = hi.next(); // Skip any fields prefixed by the underscore character _ if (!header.equals("barcode") && !header.startsWith("_")) { data.put(header, record.get(header)); if (!header.equals("questions") && MetadataRetriever.isFieldExternallyUpdatable(Specimen.class, header) && MetadataRetriever.isFieldVerbatim(Specimen.class, header)) { containsNonVerbatim = true; } } } if (data.size() > 0) { try { boolean updated = false; if (containsNonVerbatim) { updated = fl.loadFromMap(barcode, data, WorkFlowStatus.STAGE_CLASSIFIED, true); } else { updated = fl.loadFromMap(barcode, data, WorkFlowStatus.STAGE_VERBATIM, true); } counter.incrementSpecimens(); if (updated) { counter.incrementSpecimensUpdated(); } } catch (HibernateException e1) { // Catch (should just be development) problems with the underlying query StringBuilder message = new StringBuilder(); message.append("Query Error loading row (").append(lineNumber) .append(")[").append(barcode).append("]") .append(e1.getMessage()); RunnableJobError err = new RunnableJobError(selectedFilename, barcode, Integer.toString(lineNumber), e1.getMessage(), e1, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(err); log.error(e1.getMessage(), e1); } catch (LoadException e) { StringBuilder message = new StringBuilder(); message.append("Error loading row (").append(lineNumber).append(")[") .append(barcode).append("]").append(e.getMessage()); RunnableJobError err = new RunnableJobError(selectedFilename, barcode, Integer.toString(lineNumber), e.getMessage(), e, RunnableJobError.TYPE_LOAD_FAILED); counter.appendError(err); // errors.append(message.append("\n").toString()); log.error(e.getMessage(), e); } } percentComplete = (int) ((lineNumber * 100f) / rows); this.setPercentComplete(percentComplete); } } else { String message = "Can't load data, problem with headers."; errors.append(message).append("\n"); log.error(message); } } } csvParser.close(); reader.close(); } catch (FileNotFoundException e) { JOptionPane.showMessageDialog(Singleton.getSingletonInstance().getMainFrame(), "Unable to load data, file not found: " + e.getMessage(), "Error: File Not Found", JOptionPane.OK_OPTION); errors.append("File not found ").append(e.getMessage()).append("\n"); log.error(e.getMessage(), e); } catch (IOException e) { errors.append("Error Loading data: ").append(e.getMessage()).append("\n"); log.error(e.getMessage(), e); } } } else { //TODO: handle error condition log.error("File selection cancelled by user."); } report(selectedFilename); done(); }
From source file:nl.utwente.trafficanalyzer.GeoTagger.java
public static List readCsvFile(File fileName) { FileReader fileReader = null; CSVParser csvFileParser = null; //Create the CSVFormat object with the header mapping CSVFormat csvFileFormat = CSVFormat.DEFAULT; try {/*w w w. j a va2 s.c o m*/ //initialize FileReader object fileReader = new FileReader(fileName); //initialize CSVParser object csvFileParser = new CSVParser(fileReader, csvFileFormat); //Get a list of CSV file records List csvRecords = csvFileParser.getRecords(); return csvRecords; } catch (Exception e) { System.out.println("Error in CsvFileReader !!!"); e.printStackTrace(); } finally { try { fileReader.close(); csvFileParser.close(); } catch (IOException e) { System.out.println("Error while closing fileReader/csvFileParser !!!"); e.printStackTrace(); } } return null; }
From source file:no.uio.medicine.virsurveillance.parsers.CSVsGBDdata.java
public void parse(String deathFolder) throws IOException { File f = new File(deathFolder); Runtime runtime = Runtime.getRuntime(); if (f.isDirectory()) { String[] filesInDir = f.list(); for (String fil : filesInDir) { if (fil.endsWith(".zip")) { ZipFile zipFile = new ZipFile(deathFolder + "/" + fil); Enumeration<? extends ZipEntry> entries = zipFile.entries(); while (entries.hasMoreElements()) { System.out.println( "Used memory: " + (runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024) + " Free memory: " + (runtime.freeMemory()) / (1024 * 1024)); ZipEntry entry = entries.nextElement(); InputStream stream = zipFile.getInputStream(entry); BufferedReader br = new BufferedReader(new InputStreamReader(stream, "UTF-8")); CSVParser parser = CSVFormat.RFC4180.withDelimiter(',').withIgnoreEmptyLines().withHeader() .parse(br);/*from www. j av a2 s . com*/ List<CSVRecord> records = parser.getRecords(); System.out.println("Reading records: " + zipFile.getName() + "/" + entry); /*for (int i=0;i<records.size();i++) { CSVRecord csvRecord = records.get(i);*/ for (CSVRecord csvRecord : records) { if (csvRecord.isMapped("age_group_id")) { //age group 22 corresponds to all ages if (csvRecord.get("age_group_id").equalsIgnoreCase("22")) { String location = null; String year = null; String sex = null; String cause = null; String number = null; String metric = null; if (csvRecord.isMapped("location_code")) { location = csvRecord.get("location_code"); } if (csvRecord.isMapped("year")) { year = csvRecord.get("year"); } if (csvRecord.isMapped("sex_id")) { //1=male, 2 = female if (csvRecord.get("sex_id").equalsIgnoreCase(("1"))) { sex = "m"; } else if (csvRecord.get("sex_id").equalsIgnoreCase("2")) { sex = "f"; } } if (csvRecord.isMapped("cause_name")) { cause = csvRecord.get("cause_name"); } if (csvRecord.isMapped("mean")) { number = csvRecord.get("mean"); } if (csvRecord.isMapped("metric") && csvRecord.isMapped("unit")) { metric = csvRecord.get("metric") + "-" + csvRecord.get("unit"); } if (location != null && year != null && sex != null && cause != null && number != null && metric != null) { try { sqlM.addSanitaryIssueToCountry(location, year, sex, cause, metric, number); } catch (SQLException ex) { Logger.getLogger(CSVsGBDdata.class.getName()).log(Level.SEVERE, null, ex); } } } } } parser.close(); stream.close(); br.close(); } zipFile.close(); } } } else { System.out.println("Not a directory"); } }
From source file:norbert.mynemo.dataimport.fileformat.input.MovieLensIdConverter.java
/** * Loads the mapping file.//from w w w. j a va 2 s .co m * * <p> * The columns of the mapping file are: * <ol> * <li>MovieLens id of the movie</li> * <li>rating</li> * <li>average</li> * <li>IMDb id of the movie</li> * <li>title and year</li> * </ol> * * @param mappingFilepath the file that contains the mapping */ public MovieLensIdConverter(String mappingFilepath) throws IOException { checkArgument(new File(mappingFilepath).exists(), "The mapping file must exist."); CSVParser parser = new CSVParser( new CleanRatingsReader(new BufferedReader(new FileReader(mappingFilepath))), CSVFormat.MYSQL); mappings = new HashMap<>(); for (CSVRecord record : parser) { if (record.size() != RECORD_SIZE) { parser.close(); throw new IllegalStateException("Error: unable to parse the movie file \"" + mappingFilepath + "\". A list of five tab separated values is expected. Approximate" + " line number: " + record.getRecordNumber()); } mappings.put(record.get(MOVIELENS_MOVIE_ID_INDEX), record.get(IMDB_MOVIE_ID_INDEX)); } parser.close(); }
From source file:nzilbb.agcsv.AgCsvDeserializer.java
/** * Loads the serialized form of the graph, using the given set of named streams. * @param streams A list of named streams that contain all the transcription/annotation data required. * @param schema The layer schema, definining layers and the way they interrelate. * @return A list of parameters that require setting before {@link IDeserializer#deserialize()} can be invoked. This may be an empty list, and may include parameters with the value already set to a workable default. If there are parameters, and user interaction is possible, then the user may be presented with an interface for setting/confirming these parameters, before they are then passed to {@link IDeserializer#setParameters(ParameterSet)}. * @throws SerializationException If the graph could not be loaded. * @throws IOException On IO error./*w w w. j a v a 2 s . c om*/ * @throws SerializerNotConfiguredException If the configuration is not sufficient for deserialization. */ @SuppressWarnings({ "rawtypes", "unchecked" }) public ParameterSet load(NamedStream[] streams, Schema schema) throws IOException, SerializationException, SerializerNotConfiguredException { if (getFieldDelimiter() == null) throw new SerializerNotConfiguredException("fieldDelimiter must be set."); ParameterSet parameters = new ParameterSet(); // take the first csv stream, ignore all others. NamedStream csv = Utility.FindSingleStream(streams, ".csv", "text/csv"); if (csv == null) throw new SerializationException("No CSV stream found"); setName(csv.getName()); setName(getName().replaceFirst("\\.csv$", "").replaceFirst("\\.ag$", "")); reset(); CSVParser parser = new CSVParser(new InputStreamReader(csv.getStream()), CSVFormat.EXCEL.withDelimiter(fieldDelimiter.charAt(0))); mDiscoveredLayers = new HashMap<String, Layer>(); Vector<CSVRecord> vRecords = new Vector<CSVRecord>(); mCsvData.put("anchor", vRecords); // start with anchors // read all the lines, and extract the layer names for (CSVRecord line : parser) { // does it have only one field? - the layer name if (line.get(0).equals("layer")) { Layer layer = new Layer(line.get(1), line.get(2), Integer.parseInt(line.get(5)), true, // peers false, // peersOverlap false, // saturated line.get(4).equals("W") ? schema.getWordLayerId() // parentId : line.get(4).equals("M") ? schema.getTurnLayerId() // parentId : line.get(4).equals("F") ? "graph" : "segments", // parentId true); // parentIncludes int layerId = Integer.parseInt(line.get(6)); if (layerId == 11) // turn { layer.setParentId(schema.getParticipantLayerId()); } else if (layerId == 12) // utterance { layer.setSaturated(true); } else if (layerId == 0) // transcription { layer.setParentId(schema.getTurnLayerId()); } else if (layerId == 2) // orthography { layer.setPeers(false); layer.setSaturated(true); } else if (layerId == 1) // segments { layer.setSaturated(true); } layer.put("@layer_id", layerId); layer.put("@type", line.get(3)); layer.put("@scope", line.get(4)); mDiscoveredLayers.put(line.get(1), layer); Parameter p = new Parameter(layer.getId(), Layer.class, layer.getId(), layer.getDescription(), true); p.setValue(schema.getLayer(layer.getId())); p.setPossibleValues(schema.getLayers().values()); parameters.addParameter(p); // start a new set of records vRecords = new Vector<CSVRecord>(); mCsvData.put(layer.getId(), vRecords); } vRecords.add(line); } // next line parser.close(); return parameters; }
From source file:org.apache.ambari.server.api.services.serializers.CsvSerializerTest.java
@Test public void testSerializeResources_NoColumnInfo() throws Exception { Result result = new ResultImpl(true); result.setResultStatus(new ResultStatus(ResultStatus.STATUS.OK)); TreeNode<Resource> tree = result.getResultTree(); List<TreeMap<String, Object>> data = new ArrayList<TreeMap<String, Object>>() { {/*from w w w .j a va 2s. c om*/ add(new TreeMap<String, Object>() { { put("property1", "value1a"); put("property2", "value2a"); put("property3", "value3a"); put("property4", "value4a"); } }); add(new TreeMap<String, Object>() { { put("property1", "value1'b"); put("property2", "value2'b"); put("property3", "value3'b"); put("property4", "value4'b"); } }); add(new TreeMap<String, Object>() { { put("property1", "value1,c"); put("property2", "value2,c"); put("property3", "value3,c"); put("property4", "value4,c"); } }); } }; tree.setName("items"); tree.setProperty("isCollection", "true"); addChildResource(tree, "resource", 0, data.get(0)); addChildResource(tree, "resource", 1, data.get(1)); addChildResource(tree, "resource", 2, data.get(2)); replayAll(); //execute test Object o = new CsvSerializer().serialize(result).toString().replace("\r", ""); verifyAll(); assertNotNull(o); StringReader reader = new StringReader(o.toString()); CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT); List<CSVRecord> records = csvParser.getRecords(); assertNotNull(records); assertEquals(3, records.size()); int i = 0; for (CSVRecord record : records) { TreeMap<String, Object> actualData = data.get(i++); assertEquals(actualData.size(), record.size()); for (String item : record) { assertTrue(actualData.containsValue(item)); } } csvParser.close(); }
From source file:org.apache.phoenix.end2end.CSVCommonsLoaderIT.java
@Test public void testCSVCommonsUpsert() throws Exception { CSVParser parser = null; PhoenixConnection conn = null;/*from w w w . j av a 2 s . c o m*/ try { // Create table String statements = "CREATE TABLE IF NOT EXISTS " + STOCK_TABLE + "(SYMBOL VARCHAR NOT NULL PRIMARY KEY, COMPANY VARCHAR);"; conn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class); PhoenixRuntime.executeStatements(conn, new StringReader(statements), null); // Upsert CSV file CSVCommonsLoader csvUtil = new CSVCommonsLoader(conn, STOCK_TABLE, Collections.<String>emptyList(), true); csvUtil.upsert(new StringReader(STOCK_CSV_VALUES_WITH_HEADER)); // Compare Phoenix ResultSet with CSV file content PreparedStatement statement = conn.prepareStatement("SELECT SYMBOL, COMPANY FROM " + STOCK_TABLE); ResultSet phoenixResultSet = statement.executeQuery(); parser = new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_HEADER), csvUtil.getFormat()); for (CSVRecord record : parser) { assertTrue(phoenixResultSet.next()); int i = 0; for (String value : record) { assertEquals(value, phoenixResultSet.getString(i + 1)); i++; } } assertFalse(phoenixResultSet.next()); } finally { if (parser != null) parser.close(); if (conn != null) conn.close(); } }
From source file:org.apache.phoenix.end2end.CSVCommonsLoaderIT.java
@Test public void testCSVCommonsUpsert_MultiTenant() throws Exception { CSVParser parser = null; PhoenixConnection globalConn = null; PhoenixConnection tenantConn = null; try {//ww w. ja v a 2 s . c o m // Create table using the global connection String statements = "CREATE TABLE IF NOT EXISTS " + STOCK_TABLE_MULTI + "(TENANT_ID VARCHAR NOT NULL, SYMBOL VARCHAR NOT NULL, COMPANY VARCHAR," + " CONSTRAINT PK PRIMARY KEY(TENANT_ID,SYMBOL)) MULTI_TENANT = true;"; globalConn = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class); PhoenixRuntime.executeStatements(globalConn, new StringReader(statements), null); globalConn.close(); tenantConn = new PhoenixTestDriver().connect(getUrl() + ";TenantId=acme", new Properties()) .unwrap(PhoenixConnection.class); // Upsert CSV file CSVCommonsLoader csvUtil = new CSVCommonsLoader(tenantConn, STOCK_TABLE_MULTI, Collections.<String>emptyList(), true); csvUtil.upsert(new StringReader(STOCK_CSV_VALUES_WITH_HEADER)); // Compare Phoenix ResultSet with CSV file content PreparedStatement statement = tenantConn .prepareStatement("SELECT SYMBOL, COMPANY FROM " + STOCK_TABLE_MULTI); ResultSet phoenixResultSet = statement.executeQuery(); parser = new CSVParser(new StringReader(STOCK_CSV_VALUES_WITH_HEADER), csvUtil.getFormat()); for (CSVRecord record : parser) { assertTrue(phoenixResultSet.next()); int i = 0; for (String value : record) { assertEquals(value, phoenixResultSet.getString(i + 1)); i++; } } assertFalse(phoenixResultSet.next()); } finally { if (parser != null) parser.close(); if (tenantConn != null) tenantConn.close(); } }