Java tutorial
/** * Copyright (c) 2015 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.intel.databackend.datasources.hbase; import com.intel.databackend.datastructures.Observation; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.filter.ColumnPrefixFilter; import org.apache.hadoop.hbase.filter.PageFilter; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Repository; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.security.auth.login.LoginException; import java.io.IOException; import java.util.*; @Repository public class DataHbaseDao implements DataDao { private static final Logger logger = LoggerFactory.getLogger(DataHbaseDao.class); private final String tableName; private final byte[] tableNameBytes; private static final String DEVICE_MEASUREMENT = "_DEVICE_MEASUREMENT"; private Connection connection; @Autowired private HbaseConnManger hbaseConnManger; private Connection getHbaseConnection() throws IOException { try { if (connection == null || connection.isClosed()) { logger.info("Creating connection"); connection = hbaseConnManger.create(); } return connection; } catch (LoginException e) { logger.error("Unable to login into hbase", e); throw new IOException(e); } } private Table getHbaseTable() throws IOException { return getHbaseConnection().getTable(TableName.valueOf(tableNameBytes)); } @Autowired public DataHbaseDao(@Value("${vcap.application.name:local}") String hbasePrefix) { logger.info("Creating HBase. Zookeeper: "); this.tableName = hbasePrefix.toUpperCase() + DEVICE_MEASUREMENT; this.tableNameBytes = Bytes.toBytes(tableName); } @PostConstruct public boolean createTables() throws IOException { Admin admin = null; logger.info("Try to create {} in HBase.", tableName); try { admin = getHbaseConnection().getAdmin(); TableManager tableManager = new TableManager(admin, TableName.valueOf(tableNameBytes)); return tableManager.createTables(); } catch (IOException e) { logger.warn("Initialization of HBase table failed.", e); return false; } finally { if (admin != null) { admin.close(); } } } @PreDestroy public void closeHbaseConnection() throws IOException { if (connection != null && !connection.isClosed()) { connection.close(); } } @Override public boolean put(final Observation[] observations) { try (Table table = getHbaseTable()) { List<Put> puts = new ArrayList<Put>(); for (Observation obs : observations) { puts.add(getPutForObservation(obs)); } table.put(puts); } catch (IOException ex) { return false; } return true; } @Override public Observation[] scan(String accountId, String componentId, long start, long stop, Boolean gps, String[] attributes) { logger.debug("Scanning HBase: acc: {} cid: {} start: {} stop: {} gps: {}", accountId, componentId, start, stop, gps); Scan scan = new HbaseScanManager(accountId, componentId).create(start, stop).askForData(gps, attributes) .getScan(); return getObservations(accountId, componentId, gps, attributes, scan); } @Override public Observation[] scan(String accountId, String componentId, long start, long stop, Boolean gps, String[] attributes, boolean forward, int limit) { logger.debug("Scanning HBase: acc: {} cid: {} start: {} stop: {} gps: {} with limit: {}", accountId, componentId, start, stop, gps, limit); HbaseScanManager scanManager = new HbaseScanManager(accountId, componentId); if (forward) { scanManager.create(start, stop); } else { scanManager.create(stop, start).setReversed(); } scanManager.askForData(gps, attributes); logger.debug("Scanning with limit: {}", limit); Scan scan = scanManager.setCaching(limit).setFilter(new PageFilter(limit)).getScan(); return getObservations(accountId, componentId, gps, attributes, scan); } @Override public String[] scanForAttributeNames(String accountId, String componentId, long start, long stop) throws IOException { logger.debug("Scanning HBase: acc: {} cid: {} start: {} stop: {}", accountId, componentId, start, stop); Scan scan = new HbaseScanManager(accountId, componentId).create(start, stop) .setFilter(new ColumnPrefixFilter(Columns.BYTES_ATTRIBUTE_COLUMN_PREFIX)).getScan(); Set<String> attributeNames = retrieveAttributeNames(scan); return attributeNames.toArray(new String[attributeNames.size()]); } private Observation[] getObservations(final String accountId, final String componentId, final Boolean gps, final String[] attributes, Scan scan) { try (Table table = getHbaseTable(); ResultScanner scanner = table.getScanner(scan)) { List<Observation> observations = new ArrayList<>(); for (Result result : scanner) { Observation observation = new ObservationCreator(accountId, componentId).withGps(gps) .withAttributes(attributes).create(result); observations.add(observation); } return observations.toArray(new Observation[observations.size()]); } catch (IOException ex) { logger.error("Unable to find observation in hbase", ex); return null; } } private Put getPutForObservation(Observation o) { Put put = new Put(Bytes .toBytes(o.getAid() + '\0' + o.getCid() + '\0' + DataFormatter.zeroPrefixedTimestamp(o.getOn()))); put.addColumn(Columns.BYTES_COLUMN_FAMILY, Columns.BYTES_DATA_COLUMN, Bytes.toBytes(o.getValue())); if (o.getLoc() != null) { for (int i = 0; i < o.getLoc().size() && i < Columns.GPS_COLUMN_SIZE; i++) { put.addColumn(Columns.BYTES_COLUMN_FAMILY, Bytes.toBytes(DataFormatter.gpsValueToString(i)), Bytes.toBytes(o.getLoc().get(i).toString())); } } Map<String, String> attributes = o.getAttributes(); if (attributes != null) { for (String k : attributes.keySet()) { put.addColumn(Columns.BYTES_COLUMN_FAMILY, Bytes.toBytes(Columns.ATTRIBUTE_COLUMN_PREFIX + k), Bytes.toBytes(attributes.get(k))); } } return put; } private Set<String> retrieveAttributeNames(Scan scan) throws IOException { Set<String> attributes = new HashSet<>(); try (Table table = getHbaseTable(); ResultScanner scanner = table.getScanner(scan)) { for (Result result = scanner.next(); result != null; result = scanner.next()) { List<Cell> cells = result.listCells(); for (Cell cell : cells) { String attrName = DataFormatter.getAttrNameFromCell(cell); attributes.add(attrName); } } } return attributes; } }