List of usage examples for java.util NavigableMap keySet
Set<K> keySet();
From source file:com.inclouds.hbase.utils.RegionServerPoker.java
private static List<byte[]> selectRandomKeys(HTable table) throws IOException { NavigableMap<HRegionInfo, ServerName> map = table.getRegionLocations(); ServerName sn = null;/*from ww w. j a v a 2 s . c o m*/ Set<HRegionInfo> regions = null; if (seedSpecified == false && regionServer == null) { regions = map.keySet(); } else if (seedSpecified) { // select one RS sn = getServerName(map, seed); LOG.info("Selected " + sn); } else if (regionServer != null) { sn = findServerName(map, regionServer); LOG.info("Selected " + sn); } if (sn == null && (seedSpecified || regionServer != null)) { LOG.fatal("Could not select Region Server. Aborted."); System.exit(-1); } if (regions == null) { regions = regionsForServer(map, sn); } byte[] key = null; while (key == null) { HRegionInfo rInfo = select(regions); key = rInfo.getStartKey(); } List<byte[]> keys = new ArrayList<byte[]>(); Scan scan = new Scan(key); ResultScanner scanner = table.getScanner(scan); int c = 0; while (c++ < batchSize) { Result r = scanner.next(); if (r.isEmpty()) { LOG.error("Scanner result is empty"); } else { keys.add(r.getRow()); } } return keys; }
From source file:internal.coprocessor.NonAggregatingRegionObserver.java
/** * Builds the regionIndex by going through all the store files for each region and creating a reader for it that can * later be used to read the bloom filters, on demand. */// www .j a v a2 s . c o m @VisibleForTesting public static NavigableMap<ByteBuffer, ListMultimap<ByteBuffer, BloomFilter>> buildIndex(String tableName, Configuration conf, FileSystem fileSystem, Path tableBasePath) throws IOException { ImmutableSortedMap.Builder<ByteBuffer, ListMultimap<ByteBuffer, BloomFilter>> indexBuilder = ImmutableSortedMap .naturalOrder(); try { HTable table = new HTable(conf, tableName); NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations(); Collection<HColumnDescriptor> families = table.getTableDescriptor().getFamilies(); table.close(); LOG.info("Building RegionIndex [Table: " + tableName + " Base Path: " + tableBasePath.toString() + "]"); for (HRegionInfo region : regions.keySet()) { ImmutableListMultimap.Builder<ByteBuffer, BloomFilter> familiesAndStoreFiles = ImmutableListMultimap .builder(); for (HColumnDescriptor family : families) { Path path = new Path(tableBasePath, region.getEncodedName() + Path.SEPARATOR + family.getNameAsString()); FileStatus[] storeFilesStatuses = fileSystem.listStatus(path); for (FileStatus status : storeFilesStatuses) { LOG.info("Processing Store File: " + status.getPath().toString() + " Column Family: " + family.getNameAsString() + " for Region: " + region.getRegionNameAsString()); // maybe increased the cache for this reader since it's be only reading blooms? HFile.Reader reader = HFile.createReader(fileSystem, status.getPath(), new CacheConfig(conf)); familiesAndStoreFiles.put(ByteBuffer.wrap(family.getName()), BloomFilterFactory.createFromMeta(reader.getGeneralBloomFilterMetadata(), reader)); } } indexBuilder.put(ByteBuffer.wrap(region.getStartKey()), familiesAndStoreFiles.build()); } return indexBuilder.build(); } catch (Exception e) { LOG.error("Could not load regionIndex, BloomFilterSemiJoinFilter will not work.", e); } return null; }
From source file:com.moz.fiji.schema.tools.ToolUtils.java
/** * Prints cell data from the <code>row</code> for each column specified on the * <code>request</code>./*from w w w . j ava2 s. c o m*/ * * @param row The row to read from. * @param mapTypeFamilies The map type families to print. * @param groupTypeColumns The group type columns to print. * @param printStream The stream to print to. * @throws IOException if there is an error retrieving data from the FijiRowData. */ public static void printRow(FijiRowData row, Map<FamilyLayout, List<String>> mapTypeFamilies, Map<FamilyLayout, List<ColumnLayout>> groupTypeColumns, PrintStream printStream) throws IOException { // Unpack and print result for the map type families. for (Entry<FamilyLayout, List<String>> entry : mapTypeFamilies.entrySet()) { final FamilyLayout family = entry.getKey(); if (family.getDesc().getMapSchema().getType() == SchemaType.COUNTER) { // If this map family of counters has no qualifiers, print entire family. if (entry.getValue().isEmpty()) { for (String key : row.getQualifiers(family.getName())) { FijiCell<Long> counter = row.getMostRecentCell(family.getName(), key); if (null != counter) { printCell(row.getEntityId(), counter, printStream); } } // If this map family of counters has been qualified, print only the given columns. } else { for (String key : entry.getValue()) { FijiCell<Long> counter = row.getMostRecentCell(family.getName(), key); if (null != counter) { printCell(row.getEntityId(), counter, printStream); } } } } else { // If this map family of non-counters has no qualifiers, print entire family. if (entry.getValue().isEmpty()) { NavigableMap<String, NavigableMap<Long, Object>> keyTimeseriesMap = row .getValues(family.getName()); for (String key : keyTimeseriesMap.keySet()) { for (Entry<Long, Object> timestampedCell : keyTimeseriesMap.get(key).entrySet()) { long timestamp = timestampedCell.getKey(); printCell(row.getEntityId(), timestamp, family.getName(), key, timestampedCell.getValue(), printStream); } } // If this map family of non-counters has been qualified, print only the given columns. } else { for (String key : entry.getValue()) { NavigableMap<Long, Object> timeseriesMap = row.getValues(family.getName(), key); for (Entry<Long, Object> timestampedCell : timeseriesMap.entrySet()) { long timestamp = timestampedCell.getKey(); printCell(row.getEntityId(), timestamp, family.getName(), key, timestampedCell.getValue(), printStream); } } } } } // Unpack and print result for the group type families. for (Entry<FamilyLayout, List<ColumnLayout>> entry : groupTypeColumns.entrySet()) { String familyName = entry.getKey().getName(); for (ColumnLayout column : entry.getValue()) { final FijiColumnName colName = FijiColumnName.create(familyName, column.getName()); if (column.getDesc().getColumnSchema().getType() == SchemaType.COUNTER) { final FijiCell<Long> counter = row.getMostRecentCell(colName.getFamily(), colName.getQualifier()); if (null != counter) { printCell(row.getEntityId(), counter, printStream); } } else { for (Entry<Long, Object> timestampedCell : row .getValues(colName.getFamily(), colName.getQualifier()).entrySet()) { long timestamp = timestampedCell.getKey(); printCell(row.getEntityId(), timestamp, colName.getFamily(), colName.getQualifier(), timestampedCell.getValue(), printStream); } } } } printStream.println(""); }
From source file:com.ery.hadoop.mrddx.hbase.HbaseInputFormat.java
/** * ?TableRegionInfo?// w ww .j ava 2s . co m * * @param job job * @param tableName ?? * @throws IOException IO */ private static void printTableAllRegionInfo(Configuration job, String tableName) throws IOException { HTable table = new HTable(job, tableName); StringBuilder regionLog = new StringBuilder(); regionLog.append("<<Table["); regionLog.append(new String(table.getTableName())); regionLog.append("]all RegionInfo>>"); NavigableMap<HRegionInfo, ServerName> mapHRegionInfo = table.getRegionLocations(); Iterator<HRegionInfo> hregionIterator = mapHRegionInfo.keySet().iterator(); while (hregionIterator.hasNext()) { regionLog.append("\nHRegionInfo:"); HRegionInfo key = hregionIterator.next(); ServerName value = mapHRegionInfo.get(key); regionLog.append(key.toString()); regionLog.append("ServerInfo:"); regionLog.append("{name=>:"); regionLog.append(value.getServerName()); regionLog.append(" ,HostAndPort=>:"); regionLog.append(value.getHostAndPort()); regionLog.append("}"); } MRLog.info(LOG, regionLog.toString()); }
From source file:it.davidgreco.graphbase.blueprints.HBaseEdge.java
@Override public Set<String> getPropertyKeys() { try {/*from w w w . j a va 2s . c om*/ Util.EdgeIdStruct struct = Util.getEdgeIdStruct(id); Get get = new Get(struct.vertexId); Result result = graph.handle.vtable.get(get); NavigableMap<byte[], byte[]> familyMap = result .getFamilyMap(Bytes.toBytes(graph.handle.vnameEdgeProperties)); Set<String> keys = new TreeSet<String>(); Set<byte[]> bkeys = familyMap.keySet(); for (byte[] bkey : bkeys) { byte[] id = Bytes.tail(bkey, 8); if (Bytes.equals(id, struct.edgeLocalId)) { String key = Bytes.toString(Bytes.head(bkey, bkey.length - 8)); if (!key.equals("label")) keys.add(key); } } return keys; } catch (IOException e) { throw new RuntimeException(e); } }
From source file:it.davidgreco.graphbase.blueprints.HBaseVertex.java
@Override public Set<String> getPropertyKeys() { try {//from w w w . j a va 2 s. co m Get get = new Get(id); Result result = graph.handle.vtable.get(get); NavigableMap<byte[], byte[]> familyMap = result .getFamilyMap(Bytes.toBytes(graph.handle.vnameProperties)); Set<String> keys = new TreeSet<String>(); Set<byte[]> bkeys = familyMap.keySet(); for (byte[] bkey : bkeys) { if (bkey.length != 0) { keys.add(Bytes.toString(bkey)); } } return keys; } catch (IOException e) { throw new RuntimeException(e); } }
From source file:com.kylinolap.rest.service.AclService.java
private void genAces(List<Sid> sids, Result result, AclImpl acl) { List<AceInfo> aceInfos = new ArrayList<AceInfo>(); if (null != sids) { // Just return aces in sids for (Sid sid : sids) { String sidName = null; if (sid instanceof PrincipalSid) { sidName = ((PrincipalSid) sid).getPrincipal(); } else if (sid instanceof GrantedAuthoritySid) { sidName = ((GrantedAuthoritySid) sid).getGrantedAuthority(); }/*from w w w. j a v a 2s.com*/ AceInfo aceInfo = aceSerializer .deserialize(result.getValue(Bytes.toBytes(ACL_ACES_FAMILY), Bytes.toBytes(sidName))); if (null != aceInfo) { aceInfos.add(aceInfo); } } } else { NavigableMap<byte[], byte[]> familyMap = result.getFamilyMap(Bytes.toBytes(ACL_ACES_FAMILY)); for (byte[] qualifier : familyMap.keySet()) { AceInfo aceInfo = aceSerializer.deserialize(familyMap.get(qualifier)); if (null != aceInfo) { aceInfos.add(aceInfo); } } } List<AccessControlEntry> newAces = new ArrayList<AccessControlEntry>(); for (int i = 0; i < aceInfos.size(); i++) { AceInfo aceInfo = aceInfos.get(i); if (null != aceInfo) { Sid sid = aceInfo.getSidInfo().isPrincipal() ? new PrincipalSid(aceInfo.getSidInfo().getSid()) : new GrantedAuthoritySid(aceInfo.getSidInfo().getSid()); AccessControlEntry ace = new AccessControlEntryImpl(Long.valueOf(i), acl, sid, aclPermissionFactory.buildFromMask(aceInfo.getPermissionMask()), true, false, false); newAces.add(ace); } } this.setAces(acl, newAces); }
From source file:eu.ggnet.dwoss.report.eao.ReportLineEao.java
/** * Returns the revenue in a range with the supplied Stepsize. * A step size of day will return daily revenues, while a stepsize of month returns monthly revenues. * For each stepsize the earliest possible day is used as identifier. e.g.: January 2012 it would be 2012-01-01. * <p>//from ww w . j a v a2 s.co m * @param posTypes the positiontypes to include * @param start the start * @param end the end * @param step the stepsize. * @param extraReported * @return the Revenue by Date in the stepsize. */ public NavigableMap<Date, Revenue> revenueByPositionTypesAndDate(List<PositionType> posTypes, Date start, Date end, Step step, boolean extraReported) { L.debug("Attempt to find revenue report data with posType={}, start={}, end={}, {}", posTypes, start, end, step); TypedQuery<RevenueHolder> q = em.createNamedQuery("ReportLine.revenueByPositionTypesAndDate", RevenueHolder.class); q.setParameter("positions", posTypes).setParameter("start", start).setParameter("end", end); NavigableMap<Date, Revenue> result = prepare(start, end, step); for (RevenueHolder holder : q.getResultList()) { Revenue revenueStep = result.get(step.truncate(holder.getReportingDate())); // Highly unlikely case, but if it happens a detail message might help. if (revenueStep == null) throw new RuntimeException("No prepared RevenueStep found for " + step.name() + ":reportingDate=" + DateFormats.ISO.format(holder.getReportingDate()) + ",truncated=" + DateFormats.ISO.format(step.truncate(holder.getReportingDate())) + ",keys=" + nice(result.keySet(), step)); revenueStep.addTo(holder.getSalesChannel(), holder.getDocumentType(), holder.getContractor(), holder.getPrice(), 0., 0.); } if (!extraReported) return result; q = em.createNamedQuery("ReportLine.revenueByPositionTypesAndDateReported", RevenueHolder.class); q.setParameter("positions", posTypes).setParameter("start", start).setParameter("end", end); List<RevenueHolder> resultList = q.getResultList(); System.out.println("Second run size:" + resultList.size()); for (RevenueHolder holder : resultList) { System.out.println("Second run: " + holder); Revenue revenueStep = result.get(step.truncate(holder.getReportingDate())); // Highly unlikely case, but if it happens a detail message might help. if (revenueStep == null) throw new RuntimeException("No prepared RevenueStep found for " + step.name() + ":reportingDate=" + DateFormats.ISO.format(holder.getReportingDate()) + ",truncated=" + DateFormats.ISO.format(step.truncate(holder.getReportingDate())) + ",keys=" + nice(result.keySet(), step)); revenueStep.addTo(holder.getSalesChannel(), holder.getDocumentType(), holder.getContractor(), 0., holder.getPrice(), holder.getPurchasePrice()); } return result; }
From source file:com.infochimps.hadoop.pig.hbase.StaticFamilyStorage.java
@Override public Tuple getNext() throws IOException { try {//from w w w . jav a 2s . com if (!initialized) { Properties p = UDFContext.getUDFContext().getUDFProperties(this.getClass(), new String[] { contextSignature }); String projectedFields = p.getProperty(contextSignature + "_projectedFields"); if (projectedFields != null) { requiredFieldList = (RequiredFieldList) ObjectSerializer.deserialize(projectedFields); pushProjection(requiredFieldList); } initialized = true; } if (reader.nextKeyValue()) { ImmutableBytesWritable rowKey = (ImmutableBytesWritable) reader.getCurrentKey(); Result result = (Result) reader.getCurrentValue(); int tupleSize = columnInfo_.size(); // use a map of families -> qualifiers with the most recent // version of the cell. Fetching multiple vesions could be a // useful feature. NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultsMap = result.getNoVersionMap(); if (loadRowKey_) { tupleSize++; } Tuple tuple = TupleFactory.getInstance().newTuple(tupleSize); int startIndex = 0; if (loadRowKey_) { tuple.set(0, new DataByteArray(rowKey.get())); startIndex++; } for (int i = 0; i < columnInfo_.size(); ++i) { int currentIndex = startIndex + i; ColumnInfo columnInfo = columnInfo_.get(i); if (columnInfo.isColumnMap()) { // It's a column family so we need to iterate and set all // values found NavigableMap<byte[], byte[]> cfResults = resultsMap.get(columnInfo.getColumnFamily()); DataBag bagged_family = BagFactory.getInstance().newDefaultBag(); if (cfResults != null) { for (byte[] quantifier : cfResults.keySet()) { // We need to check against the prefix filter to // see if this value should be included. We can't // just rely on the server-side filter, since a // user could specify multiple CF filters for the // same CF. if (columnInfo.getColumnPrefix() == null || columnInfo.hasPrefixMatch(quantifier)) { byte[] cell = cfResults.get(quantifier); DataByteArray value = cell == null ? null : new DataByteArray(cell); Tuple entry = TupleFactory.getInstance().newTuple(2); entry.set(0, Bytes.toString(quantifier)); entry.set(1, value); bagged_family.add(entry); } } } tuple.set(currentIndex, bagged_family); } else { // It's a column so set the value if (result.containsColumn(columnInfo.getColumnFamily(), columnInfo.getColumnName())) { byte[] cell = result.getValue(columnInfo.getColumnFamily(), columnInfo.getColumnName()); DataByteArray value = (cell == null || cell.length == 0) ? new DataByteArray(ONE) : new DataByteArray(cell); tuple.set(currentIndex, value); } else { tuple.set(currentIndex, null); } } } if (LOG.isDebugEnabled()) { for (int i = 0; i < tuple.size(); i++) { LOG.debug("tuple value:" + tuple.get(i)); } } return tuple; } } catch (InterruptedException e) { throw new IOException(e); } return null; }
From source file:com.bol.crazypigs.HBaseStorage15.java
@Override public Tuple getNext() throws IOException { try {//from w w w. ja v a 2 s .co m if (reader.nextKeyValue()) { ImmutableBytesWritable rowKey = (ImmutableBytesWritable) reader.getCurrentKey(); Result result = (Result) reader.getCurrentValue(); int tupleSize = columnInfo_.size(); // use a map of families -> qualifiers with the most recent // version of the cell. Fetching multiple vesions could be a // useful feature. NavigableMap<byte[], NavigableMap<byte[], byte[]>> resultsMap = result.getNoVersionMap(); if (loadRowKey_) { tupleSize++; } Tuple tuple = TupleFactory.getInstance().newTuple(tupleSize); int startIndex = 0; if (loadRowKey_) { tuple.set(0, new DataByteArray(rowKey.get())); startIndex++; } for (int i = 0; i < columnInfo_.size(); ++i) { int currentIndex = startIndex + i; ColumnInfo columnInfo = columnInfo_.get(i); if (columnInfo.isColumnMap()) { // It's a column family so we need to iterate and set all // values found NavigableMap<byte[], byte[]> cfResults = resultsMap.get(columnInfo.getColumnFamily()); Map<String, DataByteArray> cfMap = new HashMap<String, DataByteArray>(); if (cfResults != null) { for (byte[] quantifier : cfResults.keySet()) { // We need to check against the prefix filter to // see if this value should be included. We can't // just rely on the server-side filter, since a // user could specify multiple CF filters for the // same CF. if (columnInfo.getColumnPrefix() == null || columnInfo.hasPrefixMatch(quantifier)) { byte[] cell = cfResults.get(quantifier); DataByteArray value = cell == null ? null : new DataByteArray(cell); cfMap.put(Bytes.toString(quantifier), value); } } } tuple.set(currentIndex, cfMap); } else { // It's a column so set the value byte[] cell = result.getValue(columnInfo.getColumnFamily(), columnInfo.getColumnName()); DataByteArray value = cell == null ? null : new DataByteArray(cell); tuple.set(currentIndex, value); } } if (LOG.isDebugEnabled()) { for (int i = 0; i < tuple.size(); i++) { LOG.debug("tuple value:" + tuple.get(i)); } } return tuple; } } catch (InterruptedException e) { throw new IOException(e); } return null; }