Java tutorial
/* * Copyright 2012 - 2016 Splice Machine, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use * this file except in compliance with the License. You may obtain a copy of the * License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software distributed * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR * CONDITIONS OF ANY KIND, either express or implied. See the License for the * specific language governing permissions and limitations under the License. */ package com.splicemachine.mrio.api.core; import java.io.IOException; import java.sql.SQLException; import com.splicemachine.access.api.PartitionFactory; import com.splicemachine.db.iapi.sql.execute.ExecRow; import com.splicemachine.db.iapi.types.RowLocation; import com.splicemachine.derby.impl.SpliceSpark; import com.splicemachine.si.impl.driver.SIDriver; import com.splicemachine.storage.ClientPartition; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Logger; import com.splicemachine.mrio.MRConstants; import com.splicemachine.utils.SpliceLogUtils; public class SMInputFormat extends AbstractSMInputFormat<RowLocation, ExecRow> { protected static final Logger LOG = Logger.getLogger(SMInputFormat.class); protected SMSQLUtil util; protected SMRecordReaderImpl rr; protected boolean spark; @Override public void setConf(Configuration conf) { if (LOG.isTraceEnabled()) SpliceLogUtils.trace(LOG, "setConf conf=%s", conf); this.conf = conf; String tableName = conf.get(MRConstants.SPLICE_INPUT_TABLE_NAME); String conglomerate = conf.get(MRConstants.SPLICE_INPUT_CONGLOMERATE); String tableScannerAsString = conf.get(MRConstants.SPLICE_SCAN_INFO); spark = tableScannerAsString != null; conf.setBoolean("splice.spark", spark); String jdbcString = conf.get(MRConstants.SPLICE_JDBC_STR); String rootDir = conf.get(HConstants.HBASE_DIR); if (util == null && jdbcString != null) util = SMSQLUtil.getInstance(jdbcString); if (LOG.isTraceEnabled()) SpliceLogUtils.trace(LOG, "setConf tableName=%s, conglomerate=%s, tableScannerAsString=%s" + "jdbcString=%s, rootDir=%s", tableName, conglomerate, tableScannerAsString, jdbcString, rootDir); if (conglomerate == null && !spark) { LOG.error("Conglomerate not provided when spark is activated"); throw new RuntimeException("Conglomerate not provided when spark is activated"); } if (tableName == null && conglomerate == null) { LOG.error("Table Name Supplied is null"); throw new RuntimeException("Table Name Supplied is Null"); } if (conglomerate == null) { if (jdbcString == null) { LOG.error("JDBC String Not Supplied"); throw new RuntimeException("JDBC String Not Supplied"); } try { conglomerate = util.getConglomID(tableName); conf.set(MRConstants.SPLICE_INPUT_CONGLOMERATE, conglomerate); } catch (SQLException e) { LOG.error(StringUtils.stringifyException(e)); throw new RuntimeException(e); } } try { if (SIDriver.driver() == null) SpliceSpark.setupSpliceStaticComponents(); PartitionFactory tableFactory = SIDriver.driver().getTableFactory(); setHTable(((ClientPartition) tableFactory.getTable(conglomerate)).unwrapDelegate()); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } if (tableScannerAsString == null) { if (jdbcString == null) { LOG.error("JDBC String Not Supplied"); throw new RuntimeException("JDBC String Not Supplied"); } try { conf.set(MRConstants.SPLICE_SCAN_INFO, util.getTableScannerBuilder(tableName, null).base64Encode()); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); throw new RuntimeException(e); } } if (LOG.isTraceEnabled()) SpliceLogUtils.trace(LOG, "finishingSetConf"); } public SMRecordReaderImpl getRecordReader(InputSplit split, Configuration config) throws IOException, InterruptedException { config.addResource(conf); if (LOG.isDebugEnabled()) SpliceLogUtils.debug(LOG, "getRecordReader with table=%s, inputTable=%s," + "conglomerate=%s", table, config.get(TableInputFormat.INPUT_TABLE), config.get(MRConstants.SPLICE_INPUT_CONGLOMERATE)); rr = new SMRecordReaderImpl(conf); if (table == null) { TableName tableInfo = TableName.valueOf(config.get(TableInputFormat.INPUT_TABLE)); PartitionFactory tableFactory = SIDriver.driver().getTableFactory(); table = ((ClientPartition) tableFactory.getTable(tableInfo)).unwrapDelegate(); } rr.setHTable(table); if (LOG.isDebugEnabled()) SpliceLogUtils.debug(LOG, "returning record reader"); return rr; } @Override public RecordReader<RowLocation, ExecRow> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { if (LOG.isDebugEnabled()) SpliceLogUtils.debug(LOG, "createRecordReader for split=%s, context %s", split, context); if (rr != null) return rr; return getRecordReader(split, context.getConfiguration()); } }