List of usage examples for org.apache.commons.csv CSVRecord get
public String get(final String name)
From source file:org.chanthing.csvtool.xform.ViolationDateXform.java
/** * Return a string of form mm/dd/yyyy with the violation date from the passed in record * and the license number./*w w w . j a v a 2s . c o m*/ */ public String xform(CSVRecord src) { return src.get("violDate").substring(0, 10); }
From source file:org.commonvox.hbase_column_manager.TestRepositoryAdmin.java
public void testOutputReportOnInvalidColumns(boolean useMapReduce) throws Exception { System.out.println("#testOutputReportOnInvalidColumnQualifiers " + (useMapReduce ? "USING MAPREDUCE " : "") + "has been invoked."); // environment cleanup before testing initializeTestNamespaceAndTableObjects(); clearTestingEnvironment();/*w w w . j a va 2 s. c om*/ // add schema and data to HBase // NOTE that test/resources/hbase-column-manager.xml contains wildcarded excludedTables entries Configuration configuration = MConfiguration.create(); createSchemaStructuresInHBase(configuration, false, false); createColumnDefinitions(configuration); // For NAMESPACE01_TABLE01 colFamily CF01, only valid colQualifiers are: // COLQUALIFIER01 & COLQUALIFIER02. // For NAMESPACE01_TABLE01 colFamily CF02, only valid colQualifier is: // COLQUALIFIER03. final byte[] BAD_QUALIFIER01 = Bytes.toBytes("bad_qualifier"); final byte[] BAD_QUALIFIER02 = Bytes.toBytes("very_bad_qualifier"); final byte[] GOOD_URL01 = Bytes.toBytes("https://google.com"); final byte[] BAD_URL01 = Bytes.toBytes("ftp://google.com"); final byte[] BAD_URL02 = Bytes.toBytes("another_invalid_value"); final byte[] BAD_URL03 = Bytes.toBytes("not_even_close"); try (Connection connection = MConnectionFactory.createConnection(configuration)) { try (Table namespace01Table01 = connection.getTable(NAMESPACE01_TABLE01)) { // put two rows with valid column qualifiers namespace01Table01.put(new Put(ROW_ID_04).addColumn(CF02, COLQUALIFIER03, BAD_URL01)); // invalid value namespace01Table01.put(new Put(ROW_ID_01).addColumn(CF01, COLQUALIFIER01, VALUE_2_BYTES_LONG) .addColumn(CF01, COLQUALIFIER02, VALUE_5_BYTES_LONG) .addColumn(CF02, COLQUALIFIER03, GOOD_URL01)); namespace01Table01.put(new Put(ROW_ID_02).addColumn(CF01, COLQUALIFIER01, VALUE_9_BYTES_LONG) .addColumn(CF01, COLQUALIFIER02, VALUE_82_BYTES_LONG). // invalid length addColumn(CF02, COLQUALIFIER03, BAD_URL01)); // invalid value namespace01Table01.put(new Put(ROW_ID_04).addColumn(CF02, COLQUALIFIER03, BAD_URL02)); // invalid value // put two rows with invalid column qualifiers (3 bad qualifiers in all) namespace01Table01.put(new Put(ROW_ID_03).addColumn(CF01, BAD_QUALIFIER01, VALUE_5_BYTES_LONG). // invalid qualifier addColumn(CF01, BAD_QUALIFIER02, VALUE_2_BYTES_LONG)); // invalid qualifier namespace01Table01.put(new Put(ROW_ID_04).addColumn(CF01, BAD_QUALIFIER02, VALUE_82_BYTES_LONG). // invalid qualifier addColumn(CF02, COLQUALIFIER03, BAD_URL03)); // invalid value // put one with one good column qualifier and two bad ones namespace01Table01.put(new Put(ROW_ID_05).addColumn(CF02, COLQUALIFIER02, VALUE_9_BYTES_LONG). // invalid qualifier (for CF02) addColumn(CF02, COLQUALIFIER03, VALUE_5_BYTES_LONG). // invalid VALUE!! addColumn(CF01, BAD_QUALIFIER02, VALUE_5_BYTES_LONG). // invalid qualifier addColumn(CF01, COLQUALIFIER02, VALUE_82_BYTES_LONG)); // invalid length } } // file setup final String TARGET_DIRECTORY = "target/"; // for standalone (non-JUnit) execution final String TEMP_PREFIX = "temp."; final String INVALID_COLUMNS_SUMMARY_FILE = TEMP_PREFIX + "invalidColumns.summary.csv"; final String INVALID_COLUMNS_VERBOSE_FILE = TEMP_PREFIX + "invalidColumns.verbose.csv"; final String INVALID_COLUMNS_CF01_SUMMARY_FILE = TEMP_PREFIX + "invalidColumns.table01cf01.summary.csv"; final String INVALID_COLUMNS_CF01_VERBOSE_FILE = TEMP_PREFIX + "invalidColumns.table01cf01verbose.csv"; final String INVALID_COLUMNS_CF02_SUMMARY_FILE = TEMP_PREFIX + "invalidColumns.table01cf02.summary.csv"; final String INVALID_COLUMNS_CF02_VERBOSE_FILE = TEMP_PREFIX + "invalidColumns.table01cf02verbose.csv"; final String INVALID_COLUMNS_EMPTY_TABLE = TEMP_PREFIX + "invalidColumnsEmptyTable.summary.csv"; File fileForSummaryTable01; File fileForVerboseTable01; File fileForSummaryTable01Cf01; File fileForVerboseTable01Cf01; File fileForSummaryTable01Cf02; File fileForVerboseTable01Cf02; File fileForSummaryOfEmptyTable; try { fileForSummaryTable01 = tempTestFolder.newFile(INVALID_COLUMNS_SUMMARY_FILE); fileForVerboseTable01 = tempTestFolder.newFile(INVALID_COLUMNS_VERBOSE_FILE); fileForSummaryTable01Cf01 = tempTestFolder.newFile(INVALID_COLUMNS_CF01_SUMMARY_FILE); fileForVerboseTable01Cf01 = tempTestFolder.newFile(INVALID_COLUMNS_CF01_VERBOSE_FILE); fileForSummaryTable01Cf02 = tempTestFolder.newFile(INVALID_COLUMNS_CF02_SUMMARY_FILE); fileForVerboseTable01Cf02 = tempTestFolder.newFile(INVALID_COLUMNS_CF02_VERBOSE_FILE); fileForSummaryOfEmptyTable = tempTestFolder.newFile(INVALID_COLUMNS_EMPTY_TABLE); } catch (IllegalStateException e) { // standalone (non-JUnit) execution fileForSummaryTable01 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_SUMMARY_FILE); fileForVerboseTable01 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_VERBOSE_FILE); fileForSummaryTable01Cf01 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_CF01_SUMMARY_FILE); fileForVerboseTable01Cf01 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_CF01_VERBOSE_FILE); fileForSummaryTable01Cf02 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_CF02_SUMMARY_FILE); fileForVerboseTable01Cf02 = new File(TARGET_DIRECTORY + INVALID_COLUMNS_CF02_VERBOSE_FILE); fileForSummaryOfEmptyTable = new File(TARGET_DIRECTORY + INVALID_COLUMNS_EMPTY_TABLE); } String reportGenerationFailure = COLUMN_INVALIDITY_REPORT_FAILURE + "RepositoryAdmin#outputReportOnInvalidColumnQualifiers method " + "returned unexpected boolean value"; // generate InvalidColumnQualifier reports try (Connection connection = ConnectionFactory.createConnection(configuration)) { RepositoryAdmin repositoryAdmin = new RepositoryAdmin(connection); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForSummaryTable01, NAMESPACE01_TABLE01, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForVerboseTable01, NAMESPACE01_TABLE01, true, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForSummaryTable01Cf01, NAMESPACE01_TABLE01, CF01, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForVerboseTable01Cf01, NAMESPACE01_TABLE01, CF01, true, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForSummaryTable01Cf02, NAMESPACE01_TABLE01, CF02, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnQualifiers( fileForVerboseTable01Cf02, NAMESPACE01_TABLE01, CF02, true, useMapReduce)); try { repositoryAdmin.outputReportOnInvalidColumnQualifiers(fileForSummaryOfEmptyTable, NAMESPACE02_TABLE03, false, useMapReduce); fail(reportGenerationFailure + TABLE_NOT_INCLUDED_EXCEPTION_FAILURE); } catch (TableNotIncludedForProcessingException e) { } } // read in reports and validate contents try (CSVParser parser = CSVParser.parse(fileForSummaryTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(1), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(1), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 3, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_03), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_03), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_2_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 4: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 5: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_9_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 5, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(1), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 2, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_03), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_03), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_2_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 4: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(BAD_QUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 4, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(1), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_9_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } // generate InvalidColumnLengths reports try (Connection connection = ConnectionFactory.createConnection(configuration)) { RepositoryAdmin repositoryAdmin = new RepositoryAdmin(connection); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnLengths( fileForSummaryTable01, NAMESPACE01_TABLE01, false, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnLengths( fileForVerboseTable01, NAMESPACE01_TABLE01, true, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnLengths( fileForSummaryTable01Cf01, NAMESPACE01_TABLE01, CF01, false, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnLengths( fileForVerboseTable01Cf01, NAMESPACE01_TABLE01, CF01, true, false, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnLengths(fileForSummaryTable01Cf02, NAMESPACE01_TABLE01, CF02, false, false, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnLengths(fileForVerboseTable01Cf02, NAMESPACE01_TABLE01, CF02, true, false, useMapReduce)); try { repositoryAdmin.outputReportOnInvalidColumnLengths(fileForSummaryOfEmptyTable, NAMESPACE02_TABLE03, false, false, useMapReduce); fail(reportGenerationFailure + TABLE_NOT_INCLUDED_EXCEPTION_FAILURE); } catch (TableNotIncludedForProcessingException e) { } } // read in reports and validate contents try (CSVParser parser = CSVParser.parse(fileForSummaryTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(2), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 2, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(2), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF01), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER02), record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_82_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 2, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } // generate InvalidColumnValue reports try (Connection connection = ConnectionFactory.createConnection(configuration)) { RepositoryAdmin repositoryAdmin = new RepositoryAdmin(connection); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForSummaryTable01, NAMESPACE01_TABLE01, false, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForVerboseTable01, NAMESPACE01_TABLE01, true, false, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnValues(fileForSummaryTable01Cf01, NAMESPACE01_TABLE01, CF01, false, false, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnValues(fileForVerboseTable01Cf01, NAMESPACE01_TABLE01, CF01, true, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForSummaryTable01Cf02, NAMESPACE01_TABLE01, CF02, false, false, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForVerboseTable01Cf02, NAMESPACE01_TABLE01, CF02, true, false, useMapReduce)); try { repositoryAdmin.outputReportOnInvalidColumnValues(fileForSummaryOfEmptyTable, NAMESPACE02_TABLE03, false, false, useMapReduce); fail(reportGenerationFailure + TABLE_NOT_INCLUDED_EXCEPTION_FAILURE); } catch (TableNotIncludedForProcessingException e) { } } // read in reports and validate contents try (CSVParser parser = CSVParser.parse(fileForSummaryTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL03), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 3, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL03), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 3, recordCount); } // generate InvalidColumnValue reports WITH includeAllCells == TRUE!! try (Connection connection = ConnectionFactory.createConnection(configuration)) { RepositoryAdmin repositoryAdmin = new RepositoryAdmin(connection); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForSummaryTable01, NAMESPACE01_TABLE01, false, true, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForVerboseTable01, NAMESPACE01_TABLE01, true, true, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnValues(fileForSummaryTable01Cf01, NAMESPACE01_TABLE01, CF01, false, true, useMapReduce)); assertTrue(reportGenerationFailure, !repositoryAdmin.outputReportOnInvalidColumnValues(fileForVerboseTable01Cf01, NAMESPACE01_TABLE01, CF01, true, true, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForSummaryTable01Cf02, NAMESPACE01_TABLE01, CF02, false, true, useMapReduce)); assertTrue(reportGenerationFailure, repositoryAdmin.outputReportOnInvalidColumnValues( fileForVerboseTable01Cf02, NAMESPACE01_TABLE01, CF02, true, true, useMapReduce)); try { repositoryAdmin.outputReportOnInvalidColumnValues(fileForSummaryOfEmptyTable, NAMESPACE02_TABLE03, false, true, useMapReduce); fail(reportGenerationFailure + TABLE_NOT_INCLUDED_EXCEPTION_FAILURE); } catch (TableNotIncludedForProcessingException e) { } } // read in reports and validate contents try (CSVParser parser = CSVParser.parse(fileForSummaryTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL03), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL02), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 4: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 5: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 5, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf01, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { recordCount++; } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 0, recordCount); } try (CSVParser parser = CSVParser.parse(fileForSummaryTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.SUMMARY_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { // System.out.println(record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE) + ":" // + record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE) + ":" // + record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY) + ":" // + record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER) + " " // + ColumnInvalidityReport.SummaryReportHeader.OCCURRENCES.toString() + "=" // + record.get(ColumnInvalidityReport.SummaryReportHeader.OCCURRENCES) // ); recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Occurrences-count value not as expected", String.valueOf(3), record.get(ColumnInvalidityReport.SummaryReportHeader.INVALID_OCCURRENCE_COUNT)); } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 1, recordCount); } try (CSVParser parser = CSVParser.parse(fileForVerboseTable01Cf02, StandardCharsets.UTF_8, ColumnInvalidityReport.VERBOSE_CSV_FORMAT.withSkipHeaderRecord())) { int recordCount = 0; for (CSVRecord record : parser) { // System.out.println(record.get(ColumnInvalidityReport.VerboseReportHeader.NAMESPACE) + ":" // + record.get(ColumnInvalidityReport.VerboseReportHeader.TABLE) + ":" // + record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_FAMILY) + ":" // + record.get(ColumnInvalidityReport.VerboseReportHeader.COLUMN_QUALIFIER) + " " // + ColumnInvalidityReport.VerboseReportHeader.ROW_ID.toString() + "=" // + record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID) + " " // + ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE.toString() + "=" // + record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE) // ); recordCount++; assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Namespace value not as expected", NAMESPACE01_TABLE01.getNamespaceAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.NAMESPACE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Table value not as expected", NAMESPACE01_TABLE01.getQualifierAsString(), record.get(ColumnInvalidityReport.SummaryReportHeader.TABLE)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColFamily value not as expected", Bytes.toString(CF02), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_FAMILY)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " ColQualifier value not as expected", Bytes.toString(COLQUALIFIER03), record.get(ColumnInvalidityReport.SummaryReportHeader.COLUMN_QUALIFIER)); switch (recordCount) { case 1: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_02), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 2: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL03), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 3: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL02), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 4: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_04), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(BAD_URL01), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; case 5: assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " RowId value not as expected", Bytes.toString(ROW_ID_05), record.get(ColumnInvalidityReport.VerboseReportHeader.ROW_ID)); assertEquals( COLUMN_INVALIDITY_REPORT_FAILURE + "Rec " + recordCount + " Column value not as expected", Bytes.toString(VALUE_5_BYTES_LONG), record.get(ColumnInvalidityReport.VerboseReportHeader.CELL_VALUE)); break; } } assertEquals(COLUMN_INVALIDITY_REPORT_FAILURE + "Record count in CSV file not as expected", 5, recordCount); } clearTestingEnvironment(); System.out.println("#testOutputReportOnInvalidColumnQualifiers " + (useMapReduce ? "USING MAPREDUCE " : "") + "has run to completion."); }
From source file:org.dishevelled.bio.tools.EnsemblVariantTableToVcf.java
@Override public Integer call() throws Exception { BufferedReader reader = null; PrintWriter writer = null;/*from w w w . java 2 s . c om*/ try { reader = reader(inputEnsemblVariantTableFile); writer = writer(outputVcfFile); VcfHeader header = VcfHeader.builder().withFileFormat("VCFv4.2").withMeta("##fileformat=VCFv4.2") .build(); VcfWriter.writeHeader(header, writer); List<VcfSample> samples = Collections.emptyList(); VcfWriter.writeColumnHeader(samples, writer); VcfRecord.Builder builder = VcfRecord.builder().withFilter("PASS").withFormat(""); for (CSVRecord record : CSVFormat.DEFAULT.withHeader().parse(reader)) { String variantId = record.get("Variant ID"); String[] location = record.get("Location").split(":"); String[] alleles = record.get("Alleles").replace("-", ".").split("/"); if (alleles.length != 2) { // log warning, found e.g. COSMIC_MUTATION, HGMD_MUTATION, or CCC/-/CC, G/A/C/T continue; } VcfRecord vcfRecord = builder.withChrom(location[0]).withPos(Long.parseLong(location[1])) .withRef(alleles[0]).withAlt(alleles[1]).withId(variantId).build(); VcfWriter.writeRecord(samples, vcfRecord, writer); } return 0; } finally { try { reader.close(); } catch (Exception e) { // ignore } try { writer.close(); } catch (Exception e) { // ignore } } }
From source file:org.easybatch.extensions.apache.common.csv.ApacheCommonCsvBatchReaderTest.java
@SuppressWarnings("unchecked") @Test// w w w. j ava 2s . c o m public void testBatchProcessing() throws Exception { Job job = aNewJob().reader(apacheCommonCsvBatchReader).processor(new RecordCollector()).build(); JobReport jobReport = JobExecutor.execute(job); assertThat(jobReport.getMetrics().getTotalCount()).isEqualTo(2); List<Batch> batches = (List<Batch>) jobReport.getResult(); assertThat(batches).isNotNull().hasSize(2); Batch batch1 = batches.get(0); assertThat(batch1.getPayload().size()).isEqualTo(2); CSVRecord tweet = (CSVRecord) batch1.getPayload().get(0).getPayload(); assertThat(tweet).isNotNull(); assertThat(tweet.get(0)).isEqualTo("1"); assertThat(tweet.get(1)).isEqualTo("foo"); assertThat(tweet.get(2)).isEqualTo("hello"); tweet = (CSVRecord) batch1.getPayload().get(1).getPayload(); assertThat(tweet).isNotNull(); assertThat(tweet.get(0)).isEqualTo("2"); assertThat(tweet.get(1)).isEqualTo("bar"); assertThat(tweet.get(2)).isEqualTo("hey"); Batch batch2 = batches.get(1); assertThat(batch2.getPayload().size()).isEqualTo(1); tweet = (CSVRecord) batch2.getPayload().get(0).getPayload(); assertThat(tweet).isNotNull(); assertThat(tweet.get(0)).isEqualTo("3"); assertThat(tweet.get(1)).isEqualTo("baz"); assertThat(tweet.get(2)).isEqualTo("hi"); }
From source file:org.easybatch.extensions.apache.common.csv.ApacheCommonCsvRecordReaderTest.java
@Test public void testReadNextRecord() throws Exception { recordReader.hasNextRecord();//from w ww. ja v a2 s . co m Record record = recordReader.readNextRecord(); assertThat(record).isNotNull().isInstanceOf(ApacheCommonCsvRecord.class); ApacheCommonCsvRecord apacheCommonCsvRecord = (ApacheCommonCsvRecord) record; assertThat(apacheCommonCsvRecord.getHeader()).isNotNull(); assertThat(apacheCommonCsvRecord.getHeader().getNumber()).isEqualTo(1); CSVRecord csvRecord = apacheCommonCsvRecord.getPayload(); assertThat(csvRecord.get("firstName")).isEqualTo("foo"); assertThat(csvRecord.get("lastName")).isEqualTo("bar"); assertThat(csvRecord.get("age")).isEqualTo("15"); assertThat(csvRecord.get("married")).isEqualTo("true"); }
From source file:org.eclipse.sw360.commonIO.ConvertRecord.java
public static List<RiskCategory> convertRiskCategories(List<CSVRecord> records) { ArrayList<RiskCategory> list = new ArrayList<>(records.size()); for (CSVRecord record : records) { if (record.size() < 2) break; int id = Integer.parseInt(record.get(0)); String text = record.get(1); RiskCategory category = new RiskCategory().setRiskCategoryId(id).setText(text); list.add(category);//ww w .ja v a 2 s . c om } return list; }
From source file:org.eclipse.sw360.commonIO.ConvertRecord.java
public static List<Risk> convertRisks(List<CSVRecord> records, Map<Integer, RiskCategory> categories) { List<Risk> list = new ArrayList<>(records.size()); for (CSVRecord record : records) { if (record.size() < 3) break; int id = Integer.parseInt(record.get(0)); int catId = Integer.parseInt(record.get(1)); String text = record.get(2); Risk risk = new Risk().setRiskId(id).setText(text); risk.setCategory(categories.get(catId)); list.add(risk);//from w w w. j a v a 2 s . c o m } return list; }
From source file:org.eclipse.sw360.commonIO.ConvertRecord.java
public static Map<String, Set<String>> convertCustomProperties(List<CSVRecord> records) { Map<String, Set<String>> resultProperties = new HashMap<>(); for (CSVRecord record : records) { if (!isValidPropertyRecord(record)) { break; }/* w w w .j a va 2 s . com*/ String property = record.get(1).trim(); String value = record.get(2).trim(); addPropertyAndValueToMap(property, value, resultProperties); } return resultProperties; }
From source file:org.eclipse.sw360.commonIO.ConvertRecord.java
public static Map<Integer, PropertyWithValue> convertCustomPropertiesById(List<CSVRecord> records) { Map<Integer, PropertyWithValue> resultPropertiesById = new HashMap<>(); for (CSVRecord record : records) { if (!isValidPropertyRecord(record)) { break; }// ww w. j av a2 s.c o m Integer id = Integer.parseInt(record.get(0)); String property = record.get(1).trim(); String value = record.get(2).trim(); resultPropertiesById.put(id, new PropertyWithValue(property, value)); } return resultPropertiesById; }
From source file:org.eclipse.sw360.commonIO.ConvertRecord.java
private static boolean isValidPropertyRecord(CSVRecord record) { if (record.size() < 3 || isNullEmptyOrWhitespace(record.get(1))) { return false; }//w w w.j a v a 2 s . c om try { Integer.parseInt(record.get(0)); } catch (NumberFormatException nfe) { return false; } return true; }