Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.blur.mapreduce.lib.update; import static org.junit.Assert.assertEquals; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.UUID; import org.apache.blur.MiniCluster; import org.apache.blur.mapreduce.lib.BlurRecord; import org.apache.blur.store.buffer.BufferStore; import org.apache.blur.thirdparty.thrift_0_9_0.TException; import org.apache.blur.thrift.BlurClient; import org.apache.blur.thrift.generated.Blur.Iface; import org.apache.blur.thrift.generated.BlurException; import org.apache.blur.thrift.generated.Column; import org.apache.blur.thrift.generated.ColumnDefinition; import org.apache.blur.thrift.generated.FetchResult; import org.apache.blur.thrift.generated.Record; import org.apache.blur.thrift.generated.RecordMutation; import org.apache.blur.thrift.generated.RecordMutationType; import org.apache.blur.thrift.generated.Row; import org.apache.blur.thrift.generated.RowMutation; import org.apache.blur.thrift.generated.RowMutationType; import org.apache.blur.thrift.generated.Selector; import org.apache.blur.thrift.generated.TableDescriptor; import org.apache.blur.thrift.generated.TableStats; import org.apache.blur.utils.BlurConstants; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.SequenceFile.Writer; import org.apache.hadoop.io.Text; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; public class DriverTest { private static Configuration conf = new Configuration(); private static MiniCluster miniCluster; @BeforeClass public static void setupTest() throws Exception { setupJavaHome(); File file = new File("./target/tmp/BlurInputFormatTest_tmp"); String pathStr = file.getAbsoluteFile().toURI().toString(); System.setProperty("test.build.data", pathStr + "/data"); System.setProperty("hadoop.log.dir", pathStr + "/hadoop_log"); miniCluster = new MiniCluster(); miniCluster.startBlurCluster(pathStr + "/blur", 2, 2); miniCluster.startMrMiniCluster(); conf = miniCluster.getMRConfiguration(); BufferStore.initNewBuffer(128, 128 * 128); } public static void setupJavaHome() { String str = System.getenv("JAVA_HOME"); if (str == null) { String property = System.getProperty("java.home"); if (property != null) { throw new RuntimeException("JAVA_HOME not set should probably be [" + property + "]."); } throw new RuntimeException("JAVA_HOME not set."); } } @AfterClass public static void teardown() throws IOException { if (miniCluster != null) { miniCluster.stopMrMiniCluster(); } rm(new File("build")); } private static void rm(File file) { if (!file.exists()) { return; } if (file.isDirectory()) { for (File f : file.listFiles()) { rm(f); } } file.delete(); } @Test public void testDriverAddSingleRowWithSingleRecord() throws Exception { FileSystem fileSystem = miniCluster.getFileSystem(); Path root = new Path(fileSystem.getUri() + "/"); String tableName = "testDriverAddSingleRowWithSingleRecord"; creatTable(tableName, new Path(root, "tables"), true); Driver driver = new Driver(); driver.setConf(conf); String mrIncWorkingPathStr = new Path(root, "working").toString(); generateData(mrIncWorkingPathStr); String outputPathStr = new Path(root, "output").toString(); String blurZkConnection = miniCluster.getZkConnectionString(); assertEquals(0, driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" })); Iface client = getClient(); client.loadData(tableName, outputPathStr); waitUntilAllImportsAreCompleted(client, tableName); TableStats tableStats = client.tableStats(tableName); assertEquals(1, tableStats.getRowCount()); assertEquals(1, tableStats.getRecordCount()); } @Test public void testDriverAddSingleRecordToExistingRow() throws Exception { FileSystem fileSystem = miniCluster.getFileSystem(); Path root = new Path(fileSystem.getUri() + "/"); String tableName = "testDriverAddSingleRecordToExistingRow"; Iface client = getClient(); creatTable(tableName, new Path(root, "tables"), true); addRow(client, tableName, "row1", "record1", "value1"); Driver driver = new Driver(); driver.setConf(conf); String mrIncWorkingPathStr = new Path(root, "working").toString(); generateData(mrIncWorkingPathStr); String outputPathStr = new Path(root, "output").toString(); String blurZkConnection = miniCluster.getZkConnectionString(); assertEquals(0, driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" })); client.loadData(tableName, outputPathStr); waitUntilAllImportsAreCompleted(client, tableName); TableStats tableStats = client.tableStats(tableName); assertEquals(1, tableStats.getRowCount()); assertEquals(2, tableStats.getRecordCount()); } @Test public void testDriverUpdateRecordToExistingRow() throws Exception { FileSystem fileSystem = miniCluster.getFileSystem(); Path root = new Path(fileSystem.getUri() + "/"); String tableName = "testDriverUpdateRecordToExistingRow"; Iface client = getClient(); creatTable(tableName, new Path(root, "tables"), true); String rowId = "row1"; String recordId = "record1"; addRow(client, tableName, rowId, recordId, "value1"); Driver driver = new Driver(); driver.setConf(conf); String mrIncWorkingPathStr = new Path(root, "working").toString(); generateData(mrIncWorkingPathStr, rowId, recordId, "value2"); String outputPathStr = new Path(root, "output").toString(); String blurZkConnection = miniCluster.getZkConnectionString(); assertEquals(0, driver.run(new String[] { tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1" })); { Selector selector = new Selector(); selector.setRowId(rowId); FetchResult fetchRow = client.fetchRow(tableName, selector); Row row = fetchRow.getRowResult().getRow(); assertEquals(rowId, row.getId()); List<Record> records = row.getRecords(); assertEquals(1, records.size()); Record record = records.get(0); assertEquals(recordId, record.getRecordId()); List<Column> columns = record.getColumns(); assertEquals(1, columns.size()); Column column = columns.get(0); assertEquals("col0", column.getName()); assertEquals("value1", column.getValue()); } client.loadData(tableName, outputPathStr); waitUntilAllImportsAreCompleted(client, tableName); TableStats tableStats = client.tableStats(tableName); assertEquals(1, tableStats.getRowCount()); assertEquals(1, tableStats.getRecordCount()); { Selector selector = new Selector(); selector.setRowId(rowId); FetchResult fetchRow = client.fetchRow(tableName, selector); Row row = fetchRow.getRowResult().getRow(); assertEquals(rowId, row.getId()); List<Record> records = row.getRecords(); assertEquals(1, records.size()); Record record = records.get(0); assertEquals(recordId, record.getRecordId()); List<Column> columns = record.getColumns(); assertEquals(1, columns.size()); Column column = columns.get(0); assertEquals("col0", column.getName()); assertEquals("value2", column.getValue()); } } @Test public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception { FileSystem fileSystem = miniCluster.getFileSystem(); Path root = new Path(fileSystem.getUri() + "/"); String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow"; Iface client = getClient(); Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName); creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString()); String rowId = "row1"; String recordId = "record1"; addRow(client, tableName, rowId, recordId, "value1"); generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2"); { Selector selector = new Selector(); selector.setRowId(rowId); FetchResult fetchRow = client.fetchRow(tableName, selector); Row row = fetchRow.getRowResult().getRow(); assertEquals(rowId, row.getId()); List<Record> records = row.getRecords(); assertEquals(1, records.size()); Record record = records.get(0); assertEquals(recordId, record.getRecordId()); List<Column> columns = record.getColumns(); assertEquals(1, columns.size()); Column column = columns.get(0); assertEquals("col0", column.getName()); assertEquals("value1", column.getValue()); } BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand(); bulkTableUpdateCommand.setAutoLoad(true); bulkTableUpdateCommand.setTable(tableName); bulkTableUpdateCommand.setWaitForDataBeVisible(true); bulkTableUpdateCommand.addExtraConfig(conf); assertEquals(0, (int) bulkTableUpdateCommand.run(getClient())); TableStats tableStats = client.tableStats(tableName); assertEquals(1, tableStats.getRowCount()); assertEquals(1, tableStats.getRecordCount()); { Selector selector = new Selector(); selector.setRowId(rowId); FetchResult fetchRow = client.fetchRow(tableName, selector); Row row = fetchRow.getRowResult().getRow(); assertEquals(rowId, row.getId()); List<Record> records = row.getRecords(); assertEquals(1, records.size()); Record record = records.get(0); assertEquals(recordId, record.getRecordId()); List<Column> columns = record.getColumns(); assertEquals(1, columns.size()); Column column = columns.get(0); assertEquals("col0", column.getName()); assertEquals("value2", column.getValue()); } } private void addRow(Iface client, String tableName, String rowId, String recordId, String value) throws BlurException, TException { List<RecordMutation> recordMutations = new ArrayList<RecordMutation>(); List<Column> columns = new ArrayList<Column>(); columns.add(new Column("col0", value)); Record record = new Record(recordId, "fam0", columns); recordMutations.add(new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record)); RowMutation rowMutation = new RowMutation(tableName, rowId, RowMutationType.REPLACE_ROW, recordMutations); client.mutate(rowMutation); } private void waitUntilAllImportsAreCompleted(Iface client, String tableName) throws BlurException, TException, InterruptedException { while (true) { Thread.sleep(1000); TableStats tableStats = client.tableStats(tableName); if (tableStats.getSegmentImportInProgressCount() == 0 && tableStats.getSegmentImportPendingCount() == 0) { return; } } } private void generateData(String mrIncWorkingPathStr, String rowId, String recordId, String value) throws IOException { Path path = new Path(new Path(mrIncWorkingPathStr), "new"); Writer writer = new SequenceFile.Writer(miniCluster.getFileSystem(), conf, new Path(path, UUID.randomUUID().toString()), Text.class, BlurRecord.class); BlurRecord blurRecord = new BlurRecord(); blurRecord.setRowId(rowId); blurRecord.setRecordId(recordId); blurRecord.setFamily("fam0"); blurRecord.addColumn("col0", value); writer.append(new Text(rowId), blurRecord); writer.close(); } private void generateData(String mrIncWorkingPathStr) throws IOException { generateData(mrIncWorkingPathStr, "row1", "record-" + System.currentTimeMillis(), "val0"); } private void creatTable(String tableName, Path tables, boolean fastDisable) throws BlurException, TException { creatTable(tableName, tables, fastDisable, null); } private void creatTable(String tableName, Path tables, boolean fastDisable, String workingPath) throws BlurException, TException { Path tablePath = new Path(tables, tableName); Iface client = getClient(); TableDescriptor tableDescriptor = new TableDescriptor(); tableDescriptor.setTableUri(tablePath.toString()); tableDescriptor.setName(tableName); tableDescriptor.setShardCount(2); tableDescriptor.putToTableProperties(BlurConstants.BLUR_TABLE_DISABLE_FAST_DIR, Boolean.toString(fastDisable)); if (workingPath != null) { tableDescriptor.putToTableProperties(BlurConstants.BLUR_BULK_UPDATE_WORKING_PATH, workingPath); } client.createTable(tableDescriptor); ColumnDefinition colDef = new ColumnDefinition(); colDef.setFamily("fam0"); colDef.setColumnName("col0"); colDef.setFieldType("string"); client.addColumnDefinition(tableName, colDef); } private Iface getClient() { return BlurClient.getClientFromZooKeeperConnectionStr(miniCluster.getZkConnectionString()); } }