Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package org.apache.hadoop.zebra.io; import java.io.Closeable; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.io.StringReader; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.zebra.tfile.TFile; import org.apache.hadoop.zebra.tfile.Utils; import org.apache.hadoop.zebra.tfile.MetaBlockAlreadyExists; import org.apache.hadoop.zebra.tfile.MetaBlockDoesNotExist; import org.apache.hadoop.zebra.tfile.Utils.Version; import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGRangeSplit; import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGRowSplit; import org.apache.hadoop.zebra.io.ColumnGroup.Reader.CGScanner; import org.apache.hadoop.zebra.types.CGSchema; import org.apache.hadoop.zebra.mapreduce.BasicTableOutputFormat; import org.apache.hadoop.zebra.parser.ParseException; import org.apache.hadoop.zebra.types.Partition; import org.apache.hadoop.zebra.types.Projection; import org.apache.hadoop.zebra.types.ZebraConf; import org.apache.hadoop.zebra.schema.Schema; import org.apache.hadoop.zebra.parser.TableSchemaParser; import org.apache.hadoop.zebra.pig.TableStorer; import org.apache.hadoop.zebra.types.TypesUtils; import org.apache.hadoop.zebra.types.SortInfo; import org.apache.pig.data.Tuple; /** * A materialized table that consists of one or more tightly coupled Column * Groups. * * The following Configuration parameters can customize the behavior of * BasicTable. * <ul> * <li><b>table.output.tfile.minBlock.size</b> (int) Minimum compression block * size for underlying TFile (default to 1024*1024). * <li><b>table.output.tfile.compression</b> (String) Compression method (one of * "none", "lzo", "gz") (default is "gz"). @see * {@link TFile#getSupportedCompressionAlgorithms()} * <li><b>table.input.split.minSize</b> (int) Minimum split size (default to * 64*1024). * </ul> */ public class BasicTable { static Log LOG = LogFactory.getLog(BasicTable.class); // name of the BasicTable schema file private final static String BT_SCHEMA_FILE = ".btschema"; // schema version private final static Version SCHEMA_VERSION = new Version((short) 1, (short) 1); // name of the BasicTable meta-data file private final static String BT_META_FILE = ".btmeta"; private final static String DELETED_CG_PREFIX = ".deleted-"; public final static String DELETED_CG_SEPARATOR_PER_TABLE = ","; // no public ctor for instantiating a BasicTable object private BasicTable() { // no-op } /** * Deletes the data for column group specified by cgName. * When the readers try to read the fields that were stored in the * column group get null since the underlying data is removed. * <br> <br> * * Effect on the readers that are currently reading from the table while * a column group is droped is unspecified. Suggested practice is to * drop column groups when there are no readers or writes for the table. * <br> <br> * * Column group names are usually specified in the "storage hint" while * creating a table. If no name is specified, system assigns a simple name. * These names could be obtained through "dumpInfo()" and other methods. * <br> <br> * * Dropping a column group that has already been removed is a no-op no * exception is thrown. * <br> <br> * * Note that this feature is experimental now and subject to changes in the * future. * * @param path path to BasicTable * @param conf Configuration determines file system and other parameters. * @param cgName name of the column group to drop. * @throws IOException IOException could occur for various reasons. E.g. * a user does not have permissions to write to table directory. * */ public static void dropColumnGroup(Path path, Configuration conf, String cgName) throws IOException { FileSystem fs = FileSystem.get(conf); int triedCount = 0; int numCGs = SchemaFile.getNumCGs(path, conf); SchemaFile schemaFile = null; /* Retry up to numCGs times accounting for other CG deleting threads or processes.*/ while (triedCount++ < numCGs) { try { schemaFile = new SchemaFile(path, null, conf); break; } catch (FileNotFoundException e) { LOG.info("Try " + triedCount + " times : " + e.getMessage()); } catch (Exception e) { throw new IOException("Cannot construct SchemaFile : " + e.getMessage()); } } if (schemaFile == null) { throw new IOException("Cannot construct SchemaFile"); } int cgIdx = schemaFile.getCGByName(cgName); if (cgIdx < 0) { throw new IOException(path + " : Could not find a column group with the name '" + cgName + "'"); } Path cgPath = new Path(path, schemaFile.getName(cgIdx)); //Clean up any previous unfinished attempts to drop column groups? if (schemaFile.isCGDeleted(cgIdx)) { // Clean up unfinished delete if it exists. so that clean up can // complete if the previous deletion was interrupted for some reason. if (fs.exists(cgPath)) { LOG.info(path + " : " + " clearing unfinished deletion of column group " + cgName + "."); fs.delete(cgPath, true); } LOG.info(path + " : column group " + cgName + " is already deleted."); return; } // try to delete the column group: // first check if the user has enough permissions to list the directory fs.listStatus(cgPath); //verify if the user has enough permissions by trying to create //a temporary file in cg. OutputStream out = fs.create(new Path(cgPath, ".tmp" + DELETED_CG_PREFIX + cgName), true); out.close(); //First try to create a file indicating a column group is deleted. try { Path deletedCGPath = new Path(path, DELETED_CG_PREFIX + cgName); // create without overriding. out = fs.create(deletedCGPath, false); // should we write anything? out.close(); } catch (IOException e) { // one remote possibility is that another user // already deleted CG. SchemaFile tempSchema = new SchemaFile(path, null, conf); if (tempSchema.isCGDeleted(cgIdx)) { LOG.info(path + " : " + cgName + " is deleted by someone else. That is ok."); return; } // otherwise, it is some other error. throw e; } // At this stage, the CG is marked deleted. Now just try to // delete the actual directory: if (!fs.delete(cgPath, true)) { String msg = path + " : Could not detete column group " + cgName + ". It is marked deleted."; LOG.warn(msg); throw new IOException(msg); } LOG.info("Dropped " + cgName + " from " + path); } /** * BasicTable reader. */ public static class Reader implements Closeable { private Path path; private boolean closed = true; private SchemaFile schemaFile; private Projection projection; boolean inferredMapping; private MetaFile.Reader metaReader; private BasicTableStatus status; private int firstValidCG = -1; /// First column group that exists. private int rowSplitCGIndex = -1; Partition partition; ColumnGroup.Reader[] colGroups; Tuple[] cgTuples; private synchronized void checkInferredMapping() throws ParseException, IOException { if (!inferredMapping) { for (int i = 0; i < colGroups.length; ++i) { if (colGroups[i] != null) { colGroups[i].setProjection(partition.getProjection(i)); } if (partition.isCGNeeded(i)) { if (isCGDeleted(i)) { // this is a deleted column group. Warn about it. LOG.warn("Trying to read from deleted column group " + schemaFile.getName(i) + ". NULL is returned for corresponding columns. " + "Table at " + path); } else { cgTuples[i] = TypesUtils.createTuple(colGroups[i].getSchema()); } } else cgTuples[i] = null; } partition.setSource(cgTuples); inferredMapping = true; } else { // the projection is not changed, so we do not need to recalculate the // mapping } } /** * Returns true if a column group is deleted. */ private boolean isCGDeleted(int nx) { return colGroups[nx] == null; } /** * Create a BasicTable reader. * * @param path * The directory path to the BasicTable. * @param conf * Optional configuration parameters. * @throws IOException */ public Reader(Path path, Configuration conf) throws IOException { this(path, null, conf); } public Reader(Path path, String[] deletedCGs, Configuration conf) throws IOException { try { boolean mapper = (deletedCGs != null); this.path = path; schemaFile = new SchemaFile(path, deletedCGs, conf); metaReader = MetaFile.createReader(new Path(path, BT_META_FILE), conf); // create column group readers int numCGs = schemaFile.getNumOfPhysicalSchemas(); Schema schema; colGroups = new ColumnGroup.Reader[numCGs]; cgTuples = new Tuple[numCGs]; // set default projection that contains everything schema = schemaFile.getLogical(); projection = new Projection(schema); String storage = schemaFile.getStorageString(); String comparator = schemaFile.getComparator(); partition = new Partition(schema, projection, storage, comparator); for (int nx = 0; nx < numCGs; nx++) { if (!schemaFile.isCGDeleted(nx)) { colGroups[nx] = new ColumnGroup.Reader(new Path(path, partition.getCGSchema(nx).getName()), conf, mapper); if (firstValidCG < 0) { firstValidCG = nx; } } if (colGroups[nx] != null && partition.isCGNeeded(nx)) cgTuples[nx] = TypesUtils.createTuple(colGroups[nx].getSchema()); else cgTuples[nx] = null; } closed = false; } catch (Exception e) { throw new IOException("BasicTable.Reader constructor failed : " + e.getMessage()); } finally { if (closed) { /** * Construction fails. */ if (colGroups != null) { for (int i = 0; i < colGroups.length; ++i) { if (colGroups[i] != null) { try { colGroups[i].close(); } catch (Exception e) { // ignore error } } } } if (metaReader != null) { try { metaReader.close(); } catch (Exception e) { // no-op } } } } } /** * Is the Table sorted? * * @return Whether the table is sorted. */ public boolean isSorted() { return schemaFile.isSorted(); } /** * @return the list of sorted columns */ public SortInfo getSortInfo() { return schemaFile.getSortInfo(); } /** * @return the name of i-th column group */ public String getName(int i) { return schemaFile.getName(i); } /** * Set the projection for the reader. This will affect calls to * {@link #getScanner(RangeSplit, boolean)}, * {@link #getScanner(BytesWritable, BytesWritable, boolean)}, * {@link #getStatus()}, {@link #getSchema()}. * * @param projection * The projection on the BasicTable for subsequent read operations. * For this version of implementation, the projection is a comma * separated list of column names, such as * "FirstName, LastName, Sex, Department". If we want select all * columns, pass projection==null. * @throws IOException */ public synchronized void setProjection(String projection) throws ParseException, IOException { if (projection == null) { this.projection = new Projection(schemaFile.getLogical()); partition = new Partition(schemaFile.getLogical(), this.projection, schemaFile.getStorageString(), schemaFile.getComparator()); } else { /** * the typed schema from projection which is untyped or actually typed * as "bytes" */ this.projection = new Projection(schemaFile.getLogical(), projection); partition = new Partition(schemaFile.getLogical(), this.projection, schemaFile.getStorageString(), schemaFile.getComparator()); } inferredMapping = false; } /** * Get the status of the BasicTable. */ public BasicTableStatus getStatus() throws IOException { if (status == null) buildStatus(); return status; } /** * Given a split range, calculate how the file data that fall into the range * are distributed among hosts. * * @param split * The range-based split. Can be null to indicate the whole TFile. * @return An object that conveys how blocks fall in the split are * distributed across hosts. * @see #rangeSplit(int) */ public BlockDistribution getBlockDistribution(RangeSplit split) throws IOException { BlockDistribution bd = new BlockDistribution(); if (firstValidCG >= 0) { for (int nx = 0; nx < colGroups.length; nx++) { if (partition.isCGNeeded(nx) && !isCGDeleted(nx)) { bd.add(colGroups[nx].getBlockDistribution(split == null ? null : split.getCGRangeSplit())); } } } return bd; } /** * Given a row-based split, calculate how the file data that fall into the split * are distributed among hosts. * * @param split The row-based split. <i>Cannot</i> be null. * @return An object that conveys how blocks fall into the split are * distributed across hosts. */ public BlockDistribution getBlockDistribution(RowSplit split) throws IOException { BlockDistribution bd = new BlockDistribution(); int cgIdx = split.getCGIndex(); bd.add(colGroups[cgIdx].getBlockDistribution(split.getCGRowSplit())); return bd; } /** * Collect some key samples and use them to partition the table. Only * applicable to sorted BasicTable. The returned {@link KeyDistribution} * object also contains information on how data are distributed for each * key-partitioned bucket. * * @param n * Targeted size of the sampling. * @param nTables * Number of tables in union * @return KeyDistribution object. * @throws IOException */ public KeyDistribution getKeyDistribution(int n, int nTables, BlockDistribution lastBd) throws IOException { if (firstValidCG >= 0) { // pick the largest CG as in the row split case return colGroups[getRowSplitCGIndex()].getKeyDistribution(n, nTables, lastBd); } return null; } /** * Get a scanner that reads all rows whose row keys fall in a specific * range. Only applicable to sorted BasicTable. * * @param beginKey * The begin key of the scan range. If null, start from the first * row in the table. * @param endKey * The end key of the scan range. If null, scan till the last row * in the table. * @param closeReader * close the underlying Reader object when we close the scanner. * Should be set to true if we have only one scanner on top of the * reader, so that we should release resources after the scanner is * closed. * @return A scanner object. * @throws IOException */ public synchronized TableScanner getScanner(BytesWritable beginKey, BytesWritable endKey, boolean closeReader) throws IOException { try { checkInferredMapping(); } catch (Exception e) { throw new IOException("getScanner failed : " + e.getMessage()); } return new BTScanner(beginKey, endKey, closeReader, partition); } /** * Get a scanner that reads a consecutive number of rows as defined in the * {@link RangeSplit} object, which should be obtained from previous calls * of {@link #rangeSplit(int)}. * * @param split * The split range. If null, get a scanner to read the complete * table. * @param closeReader * close the underlying Reader object when we close the scanner. * Should be set to true if we have only one scanner on top of the * reader, so that we should release resources after the scanner is * closed. * @return A scanner object. * @throws IOException */ public synchronized TableScanner getScanner(RangeSplit split, boolean closeReader) throws IOException, ParseException { checkInferredMapping(); return new BTScanner(split, partition, closeReader); } /** * Get a scanner that reads a consecutive number of rows as defined in the * {@link RowSplit} object. * * @param closeReader * close the underlying Reader object when we close the scanner. * Should be set to true if we have only one scanner on top of the * reader, so that we should release resources after the scanner is * closed. * @param rowSplit split based on row numbers. * * @return A scanner object. * @throws IOException */ public synchronized TableScanner getScanner(boolean closeReader, RowSplit rowSplit) throws IOException, ParseException, ParseException { checkInferredMapping(); return new BTScanner(rowSplit, closeReader, partition); } /** * Get the schema of the table. The schema may be different from * {@link BasicTable.Reader#getSchema(Path, Configuration)} if a projection * has been set on the table. * * @return The schema of the BasicTable. */ public Schema getSchema() { return projection.getSchema(); } /** * Get the BasicTable schema without loading the full table index. * * @param path * The path to the BasicTable. * @deletedCGs * The deleted column groups from front end; null if unavailable from front end * @param conf * @return The logical Schema of the table (all columns). * @throws IOException */ public static Schema getSchema(Path path, Configuration conf) throws IOException { // fake an empty deleted cg list as getSchema does not care about deleted cgs SchemaFile schF = new SchemaFile(path, new String[0], conf); return schF.getLogical(); } /** * Get the path to the table. * * @return The path string to the table. */ public String getPath() { return path.toString(); } /** * Get the path filter used by the table. */ public PathFilter getPathFilter(Configuration conf) { ColumnGroup.CGPathFilter filter = new ColumnGroup.CGPathFilter(); ColumnGroup.CGPathFilter.setConf(conf); return filter; } /** * Split the table into at most n parts. * * @param n Maximum number of parts in the output list. * @return A list of RangeSplit objects, each of which can be used to * construct TableScanner later. */ public List<RangeSplit> rangeSplit(int n) throws IOException { // use the first non-deleted column group to do split, other column groups will be split exactly the same way. List<RangeSplit> ret; if (firstValidCG >= 0) { List<CGRangeSplit> cgSplits = colGroups[firstValidCG].rangeSplit(n); int numSlices = cgSplits.size(); ret = new ArrayList<RangeSplit>(numSlices); for (int slice = 0; slice < numSlices; slice++) { CGRangeSplit oneSliceSplit = cgSplits.get(slice); ret.add(new BasicTable.Reader.RangeSplit(oneSliceSplit)); } return ret; } else { // all column groups are dropped. ret = new ArrayList<RangeSplit>(1); // add a dummy split ret.add(new BasicTable.Reader.RangeSplit(new CGRangeSplit(0, 0))); return ret; } } /** * We already use FileInputFormat to create byte offset-based input splits. * Their information is encoded in starts, lengths and paths. This method is * to wrap this information to form RowSplit objects at basic table level. * * @param starts array of starting byte of fileSplits. * @param lengths array of length of fileSplits. * @param paths array of path of fileSplits. * @param splitCGIndex index of column group that is used to create fileSplits. * @return A list of RowSplit objects, each of which can be used to * construct a TableScanner later. * */ public List<RowSplit> rowSplit(long[] starts, long[] lengths, Path[] paths, int splitCGIndex, int[] batchSizes, int numBatches) throws IOException { List<RowSplit> ret; List<CGRowSplit> cgSplits = colGroups[splitCGIndex].rowSplit(starts, lengths, paths, batchSizes, numBatches); int numSlices = cgSplits.size(); ret = new ArrayList<RowSplit>(numSlices); for (int slice = 0; slice < numSlices; slice++) { CGRowSplit cgRowSplit = cgSplits.get(slice); ret.add(new BasicTable.Reader.RowSplit(splitCGIndex, cgRowSplit)); } return ret; } /** * Rearrange the files according to the column group index ordering * * @param filestatus array of FileStatus to be rearraged on */ public void rearrangeFileIndices(FileStatus[] fileStatus) throws IOException { colGroups[getRowSplitCGIndex()].rearrangeFileIndices(fileStatus); } /** * Get index of the column group that will be used for row-based split. * */ public int getRowSplitCGIndex() throws IOException { // Try to find the largest non-deleted and used column group by projection; // Try to find the largest non-deleted and used column group by projection; if (rowSplitCGIndex == -1) { int largestCGIndex = -1; long largestCGSize = -1; for (int i = 0; i < colGroups.length; i++) { if (!partition.isCGNeeded(i) || isCGDeleted(i)) { continue; } ColumnGroup.Reader reader = colGroups[i]; BasicTableStatus btStatus = reader.getStatus(); long size = btStatus.getSize(); if (size > largestCGSize) { largestCGIndex = i; largestCGSize = size; } } /* We do have a largest non-deleted and used column group, and we use it to do split. */ if (largestCGIndex >= 0) { rowSplitCGIndex = largestCGIndex; } else if (firstValidCG >= 0) { /* If all projection columns are either deleted or non-existing, then we use the first non-deleted column group to do split if it exists. */ rowSplitCGIndex = firstValidCG; } } return rowSplitCGIndex; } /** * Close the BasicTable for reading. Resources are released. */ @Override public void close() throws IOException { if (!closed) { try { closed = true; metaReader.close(); for (int i = 0; i < colGroups.length; ++i) { if (colGroups[i] != null) { colGroups[i].close(); } } } finally { try { metaReader.close(); } catch (Exception e) { // no-op } for (int i = 0; i < colGroups.length; ++i) { try { colGroups[i].close(); } catch (Exception e) { // no-op } } } } } String getBTSchemaString() { return schemaFile.getBTSchemaString(); } String getStorageString() { return schemaFile.getStorageString(); } public String getDeletedCGs() { return schemaFile.getDeletedCGs(); } public static String getDeletedCGs(Path path, Configuration conf) throws IOException { SchemaFile schF = new SchemaFile(path, new String[0], conf); return schF.getDeletedCGs(); } private void buildStatus() throws IOException { status = new BasicTableStatus(); if (firstValidCG >= 0) { status.beginKey = colGroups[firstValidCG].getStatus().getBeginKey(); status.endKey = colGroups[firstValidCG].getStatus().getEndKey(); status.rows = colGroups[firstValidCG].getStatus().getRows(); } else { status.beginKey = new BytesWritable(new byte[0]); status.endKey = status.beginKey; status.rows = 0; } status.size = 0; for (int nx = 0; nx < colGroups.length; nx++) { if (colGroups[nx] != null) { status.size += colGroups[nx].getStatus().getSize(); } } } /** * Obtain an input stream for reading a meta block. * * @param name * The name of the meta block. * @return The input stream for reading the meta block. * @throws IOException * @throws MetaBlockDoesNotExist */ public DataInputStream getMetaBlock(String name) throws MetaBlockDoesNotExist, IOException { return metaReader.getMetaBlock(name); } /** * A range-based split on the metaReadertable.The content of the split is * implementation-dependent. */ public static class RangeSplit implements Writable { //CGRangeSplit[] slice; CGRangeSplit slice; RangeSplit(CGRangeSplit split) { slice = split; } /** * Default constructor. */ public RangeSplit() { // no-op } /** * @see Writable#readFields(DataInput) */ @Override public void readFields(DataInput in) throws IOException { for (int nx = 0; nx < 1; nx++) { CGRangeSplit cgrs = new CGRangeSplit(); cgrs.readFields(in); slice = cgrs; } } /** * @see Writable#write(DataOutput) */ @Override public void write(DataOutput out) throws IOException { //Utils.writeVInt(out, slice.length); //for (CGRangeSplit split : slice) { // split.write(out); //} slice.write(out); } //CGRangeSplit get(int index) { // return slice[index]; //} CGRangeSplit getCGRangeSplit() { return slice; } } /** * A row-based split on the zebra table; */ public static class RowSplit implements Writable { int cgIndex; // column group index where split lies on; CGRowSplit slice; RowSplit(int cgidx, CGRowSplit split) { this.cgIndex = cgidx; this.slice = split; } /** * Default constructor. */ public RowSplit() { // no-op } @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{cgIndex = " + cgIndex + "}\n"); sb.append(slice.toString()); return sb.toString(); } /** * @see Writable#readFields(DataInput) */ @Override public void readFields(DataInput in) throws IOException { this.cgIndex = Utils.readVInt(in); CGRowSplit cgrs = new CGRowSplit(); cgrs.readFields(in); this.slice = cgrs; } /** * @see Writable#write(DataOutput) */ @Override public void write(DataOutput out) throws IOException { Utils.writeVInt(out, cgIndex); slice.write(out); } int getCGIndex() { return cgIndex; } CGRowSplit getCGRowSplit() { return slice; } } /** * BasicTable scanner class */ private class BTScanner implements TableScanner { private Projection schema; private CGScanner[] cgScanners; private int opCount = 0; Random random = new Random(System.nanoTime()); // checking for consistency once every 1000 times. private static final int VERIFY_FREQ = 1000; private boolean sClosed = false; private boolean closeReader; private Partition partition; private synchronized boolean checkIntegrity() { return ((++opCount % VERIFY_FREQ) == 0) && (cgScanners.length > 1); } public BTScanner(BytesWritable beginKey, BytesWritable endKey, boolean closeReader, Partition partition) throws IOException { init(null, null, beginKey, endKey, closeReader, partition); } public BTScanner(RangeSplit split, Partition partition, boolean closeReader) throws IOException { init(null, split, null, null, closeReader, partition); } public BTScanner(RowSplit rowSplit, boolean closeReader, Partition partition) throws IOException { init(rowSplit, null, null, null, closeReader, partition); } /** * Creates new CGRowSplit. If the startRow in rowSplit is not set * (i.e. < 0), it sets the startRow and numRows based on 'startByte' * and 'numBytes' from given rowSplit. */ private CGRowSplit makeCGRowSplit(RowSplit rowSplit) throws IOException { CGRowSplit inputCGSplit = rowSplit.getCGRowSplit(); int cgIdx = rowSplit.getCGIndex(); CGRowSplit cgSplit = new CGRowSplit(); // Find the row range : if (isCGDeleted(cgIdx)) { throw new IOException("CG " + cgIdx + " is deleted."); } //fill the row numbers. colGroups[cgIdx].fillRowSplit(cgSplit, inputCGSplit); return cgSplit; } // Helper function for initialization. private CGScanner createCGScanner(int cgIndex, CGRowSplit cgRowSplit, RangeSplit rangeSplit, BytesWritable beginKey, BytesWritable endKey) throws IOException, ParseException, ParseException { if (cgRowSplit != null) { return colGroups[cgIndex].getScanner(false, cgRowSplit); } if (beginKey != null || endKey != null) { return colGroups[cgIndex].getScanner(beginKey, endKey, false); } return colGroups[cgIndex].getScanner((rangeSplit == null ? null : rangeSplit.getCGRangeSplit()), false); } /** * If rowRange is not null, scanners will be created based on the * row range. <br> * If RangeSplit is not null, scaller will be based on the range, <br> * otherwise, these are based on keys. */ private void init(RowSplit rowSplit, RangeSplit rangeSplit, BytesWritable beginKey, BytesWritable endKey, boolean closeReader, Partition partition) throws IOException { this.partition = partition; boolean anyScanner = false; CGRowSplit cgRowSplit = null; if (rowSplit != null) { cgRowSplit = makeCGRowSplit(rowSplit); } try { schema = partition.getProjection(); cgScanners = new CGScanner[colGroups.length]; for (int i = 0; i < colGroups.length; ++i) { if (!isCGDeleted(i) && partition.isCGNeeded(i)) { anyScanner = true; cgScanners[i] = createCGScanner(i, cgRowSplit, rangeSplit, beginKey, endKey); } else cgScanners[i] = null; } if (!anyScanner && firstValidCG >= 0) { // if no CG is needed explicitly by projection but the "countRow" still needs to access some column group cgScanners[firstValidCG] = createCGScanner(firstValidCG, cgRowSplit, rangeSplit, beginKey, endKey); } this.closeReader = closeReader; sClosed = false; } catch (Exception e) { throw new IOException("BTScanner constructor failed : " + e.getMessage()); } finally { if (sClosed) { if (cgScanners != null) { for (int i = 0; i < cgScanners.length; ++i) { if (cgScanners[i] != null) { try { cgScanners[i].close(); cgScanners[i] = null; } catch (Exception e) { // no-op } } } } } } } @Override public boolean advance() throws IOException { boolean first = false, cur, firstAdvance = true; for (int nx = 0; nx < cgScanners.length; nx++) { if (cgScanners[nx] != null) { cur = cgScanners[nx].advanceCG(); if (!firstAdvance) { if (cur != first) { throw new IOException("advance() failed: Column Groups are not evenly positioned."); } } else { firstAdvance = false; first = cur; } } } return first; } @Override public boolean atEnd() throws IOException { boolean ret = true; int i; for (i = 0; i < cgScanners.length; i++) { if (cgScanners[i] != null) { ret = cgScanners[i].atEnd(); break; } } if (i == cgScanners.length) { return true; } if (!checkIntegrity()) { return ret; } while (true) { int index = random.nextInt(cgScanners.length); if (cgScanners[index] != null) { if (cgScanners[index].atEnd() != ret) { throw new IOException("atEnd() failed: Column Groups are not evenly positioned."); } break; } } return ret; } @Override public void getKey(BytesWritable key) throws IOException { int i; for (i = 0; i < cgScanners.length; i++) { if (cgScanners[i] != null) { cgScanners[i].getCGKey(key); break; } } if (i == cgScanners.length) return; if (!checkIntegrity()) { return; } while (true) { int index = random.nextInt(cgScanners.length); if (cgScanners[index] != null) { BytesWritable key2 = new BytesWritable(); cgScanners[index].getCGKey(key2); if (key.equals(key2)) { return; } break; } } throw new IOException("getKey() failed: Column Groups are not evenly positioned."); } @Override public void getValue(Tuple row) throws IOException { if (row.size() < projection.getSchema().getNumColumns()) { throw new IOException("Mismatched tuple object"); } for (int i = 0; i < cgScanners.length; ++i) { if (cgScanners[i] != null) { if (partition.isCGNeeded(i)) { if (cgTuples[i] == null) throw new AssertionError("cgTuples[" + i + "] is null"); cgScanners[i].getCGValue(cgTuples[i]); } } } try { partition.read(row); } catch (Exception e) { throw new IOException("getValue() failed: " + e.getMessage()); } } @Override public boolean seekTo(BytesWritable key) throws IOException { boolean first = false, cur, firstset = false; for (int nx = 0; nx < cgScanners.length; nx++) { if (cgScanners[nx] == null) continue; cur = cgScanners[nx].seekTo(key); if (firstset) { if (cur != first) { throw new IOException("seekTo() failed: Column Groups are not evenly positioned."); } } else { first = cur; firstset = true; } } return first; } @Override public void seekToEnd() throws IOException { for (int nx = 0; nx < cgScanners.length; nx++) { if (cgScanners[nx] == null) continue; cgScanners[nx].seekToEnd(); } } @Override public String getProjection() { return schema.toString(); } @Override public Schema getSchema() { return schema.getSchema(); } @Override public void close() throws IOException { if (sClosed) return; sClosed = true; try { for (int nx = 0; nx < cgScanners.length; nx++) { if (cgScanners[nx] == null) continue; cgScanners[nx].close(); cgScanners[nx] = null; } if (closeReader) { BasicTable.Reader.this.close(); } } finally { for (int nx = 0; nx < cgScanners.length; nx++) { if (cgScanners[nx] == null) continue; try { cgScanners[nx].close(); cgScanners[nx] = null; } catch (Exception e) { // no-op } } if (closeReader) { try { BasicTable.Reader.this.close(); } catch (Exception e) { // no-op } } } } } } /** * BasicTable writer. */ public static class Writer implements Closeable { private SchemaFile schemaFile; private MetaFile.Writer metaWriter; private boolean closed = true; ColumnGroup.Writer[] colGroups; Partition partition; boolean sorted; private boolean finished; Tuple[] cgTuples; private Path actualOutputPath; private Configuration writerConf; /** * Create a BasicTable writer. The semantics are as follows: * <ol> * <li>If path does not exist: * <ul> * <li>create the path directory, and initialize the directory for future * row insertion.. * </ul> * <li>If path exists and the directory is empty: initialize the directory * for future row insertion. * <li>If path exists and contains what look like a complete BasicTable, * IOException will be thrown. * </ol> * This constructor never removes a valid/complete BasicTable. * * @param path * The path to the Basic Table, either not existent or must be a * directory. * @param btSchemaString * The schema of the Basic Table. For this version of * implementation, the schema of a table is a comma or * semicolon-separated list of column names, such as * "FirstName, LastName; Sex, Department". * @param sortColumns * String of comma-separated sorted columns: null for unsorted tables * @param comparator * Name of the comparator used in sorted tables * @param conf * Optional Configuration objects. * * @throws IOException * @see Schema */ public Writer(Path path, String btSchemaString, String btStorageString, String sortColumns, String comparator, Configuration conf) throws IOException { try { actualOutputPath = path; writerConf = conf; schemaFile = new SchemaFile(path, btSchemaString, btStorageString, sortColumns, comparator, conf); partition = schemaFile.getPartition(); int numCGs = schemaFile.getNumOfPhysicalSchemas(); colGroups = new ColumnGroup.Writer[numCGs]; cgTuples = new Tuple[numCGs]; sorted = schemaFile.isSorted(); for (int nx = 0; nx < numCGs; nx++) { colGroups[nx] = new ColumnGroup.Writer(new Path(path, schemaFile.getName(nx)), schemaFile.getPhysicalSchema(nx), sorted, comparator, schemaFile.getName(nx), schemaFile.getSerializer(nx), schemaFile.getCompressor(nx), schemaFile.getOwner(nx), schemaFile.getGroup(nx), schemaFile.getPerm(nx), false, conf); cgTuples[nx] = TypesUtils.createTuple(colGroups[nx].getSchema()); } metaWriter = MetaFile.createWriter(new Path(path, BT_META_FILE), conf); partition.setSource(cgTuples); closed = false; } catch (Exception e) { throw new IOException("ColumnGroup.Writer constructor failed : " + e.getMessage()); } finally { ; if (!closed) return; if (metaWriter != null) { try { metaWriter.close(); } catch (Exception e) { // no-op } } if (colGroups != null) { for (int i = 0; i < colGroups.length; ++i) { if (colGroups[i] != null) { try { colGroups[i].close(); } catch (Exception e) { // no-op } } } } } } /** * a wrapper to support backward compatible constructor */ public Writer(Path path, String btSchemaString, String btStorageString, Configuration conf) throws IOException { this(path, btSchemaString, btStorageString, null, null, conf); } /** * Reopen an already created BasicTable for writing. Exception will be * thrown if the table is already closed, or is in the process of being * closed. */ public Writer(Path path, Configuration conf) throws IOException { try { actualOutputPath = path; writerConf = conf; if (ZebraConf.getOutputSchema(conf) != null) { schemaFile = new SchemaFile(conf); // Read out schemaFile from conf, instead of from hdfs; } else { // This is only for io test cases and it cannot happen for m/r and pig cases; schemaFile = new SchemaFile(path, new String[0], conf); // fake an empty deleted cg list as no cg should have been deleted now } int numCGs = schemaFile.getNumOfPhysicalSchemas(); partition = schemaFile.getPartition(); sorted = schemaFile.isSorted(); colGroups = new ColumnGroup.Writer[numCGs]; cgTuples = new Tuple[numCGs]; Path tmpWorkPath = new Path(path, "_temporary"); for (int nx = 0; nx < numCGs; nx++) { CGSchema cgschema = new CGSchema(schemaFile.getPhysicalSchema(nx), sorted, schemaFile.getComparator(), schemaFile.getName(nx), schemaFile.getSerializer(nx), schemaFile.getCompressor(nx), schemaFile.getOwner(nx), schemaFile.getGroup(nx), schemaFile.getPerm(nx)); colGroups[nx] = new ColumnGroup.Writer(new Path(path, partition.getCGSchema(nx).getName()), new Path(tmpWorkPath, partition.getCGSchema(nx).getName()), cgschema, conf); cgTuples[nx] = TypesUtils.createTuple(colGroups[nx].getSchema()); } partition.setSource(cgTuples); metaWriter = MetaFile.createWriter(new Path(path, BT_META_FILE), conf); closed = false; } catch (Exception e) { throw new IOException("ColumnGroup.Writer failed : " + e.getMessage()); } finally { if (!closed) return; if (metaWriter != null) { try { metaWriter.close(); } catch (Exception e) { // no-op } } if (colGroups != null) { for (int i = 0; i < colGroups.length; ++i) { if (colGroups[i] != null) { try { colGroups[i].close(); } catch (Exception e) { // no-op } } } } } } /** * Release resources used by the object. Unlike close(), finish() does not * make the table immutable. */ public void finish() throws IOException { if (finished) return; finished = true; try { for (int nx = 0; nx < colGroups.length; nx++) { if (colGroups[nx] != null) { colGroups[nx].finish(); } } metaWriter.finish(); } finally { try { metaWriter.finish(); } catch (Exception e) { // no-op } for (int i = 0; i < colGroups.length; ++i) { try { colGroups[i].finish(); } catch (Exception e) { // no-op } } } } /** * Close the BasicTable for writing. No more inserters can be obtained after * close(). */ @Override public void close() throws IOException { cleanupTempDir(); if (closed) return; closed = true; if (!finished) finish(); try { ColumnGroup.CGIndex firstCGIndex = null, cgIndex; int first = -1; for (int nx = 0; nx < colGroups.length; nx++) { if (colGroups[nx] != null) { colGroups[nx].close(); if (first == -1) { first = nx; firstCGIndex = colGroups[nx].index; } else { cgIndex = colGroups[nx].index; if (cgIndex.size() != firstCGIndex.size()) throw new IOException("Column Group " + colGroups[nx].path.getName() + " has different number of files than in column group " + colGroups[first].path.getName()); int size = firstCGIndex.size(); for (int i = 0; i < size; i++) { if (!cgIndex.get(i).name.equals(firstCGIndex.get(i).name)) throw new IOException("File[" + i + "] in Column Group " + colGroups[nx].path.getName() + " has a different name: " + cgIndex.get(i).name + " than " + firstCGIndex.get(i).name + " in column group " + colGroups[first].path.getName()); if (cgIndex.get(i).rows != firstCGIndex.get(i).rows) throw new IOException("File " + cgIndex.get(i).name + "Column Group " + colGroups[nx].path.getName() + " has a different number of rows, " + cgIndex.get(i).rows + ", than " + firstCGIndex.get(i).rows + " in column group " + colGroups[first].path.getName()); } } } } metaWriter.close(); } finally { try { metaWriter.close(); } catch (Exception e) { // no-op } for (int i = 0; i < colGroups.length; ++i) { try { colGroups[i].close(); } catch (Exception e) { // no-op } } } } /** * Removes the temporary directory underneath * $path/_temporary used to create intermediate data * during recrd writing */ private void cleanupTempDir() throws IOException { FileSystem fileSys = actualOutputPath.getFileSystem(writerConf); Path pathToRemove = new Path(actualOutputPath, "_temporary"); if (fileSys.exists(pathToRemove)) { if (!fileSys.delete(pathToRemove, true)) { LOG.error("Failed to delete the temporary output" + " directory: " + pathToRemove.toString()); } } } /** * Get the schema of the table. * * @return the Schema object. */ public Schema getSchema() { return schemaFile.getLogical(); } /** * @return sortness */ public boolean isSorted() { return sorted; } /** * Get the list of sorted columns. * @return the list of sorted columns */ public SortInfo getSortInfo() { return schemaFile.getSortInfo(); } /** * Get a inserter with a given name. * * @param name * the name of the inserter. If multiple calls to getInserter with * the same name has been called, we expect they are the result of * speculative execution and at most one of them will succeed. * @param finishWriter * finish the underlying Writer object upon the close of the * Inserter. Should be set to true if there is only one inserter * operate on the table, so we should call finish() after the * Inserter is closed. * * @return A inserter object. * @throws IOException */ public TableInserter getInserter(String name, boolean finishWriter) throws IOException { return this.getInserter(name, finishWriter, true); } /** * Get a inserter with a given name. * * @param name * the name of the inserter. If multiple calls to getInserter with * the same name has been called, we expect they are the result of * speculative execution and at most one of them will succeed. * @param finishWriter * finish the underlying Writer object upon the close of the * Inserter. Should be set to true if there is only one inserter * operate on the table, so we should call finish() after the * Inserter is closed. * @param checktype * whether or not do type check. * * @return A inserter object. * @throws IOException */ public TableInserter getInserter(String name, boolean finishWriter, boolean checkType) throws IOException { if (closed) { throw new IOException("BasicTable closed"); } return new BTInserter(name, finishWriter, partition, checkType); } /** * Obtain an output stream for creating a Meta Block with the specific name. * This method can only be called after we insert all rows into the table. * All Meta Blocks must be created by a single process prior to closing the * table. No more inserter can be created after this call. * * @param name * The name of the Meta Block * @return The output stream. Close the stream to conclude the writing. * @throws IOException * @throws MetaBlockAlreadyExists */ public DataOutputStream createMetaBlock(String name) throws MetaBlockAlreadyExists, IOException { return metaWriter.createMetaBlock(name); } private class BTInserter implements TableInserter { private TableInserter cgInserters[]; private boolean sClosed = true; private boolean finishWriter; private Partition partition = null; BTInserter(String name, boolean finishWriter, Partition partition) throws IOException { this(name, finishWriter, partition, true); } BTInserter(String name, boolean finishWriter, Partition partition, boolean checkType) throws IOException { try { cgInserters = new ColumnGroup.Writer.CGInserter[colGroups.length]; for (int nx = 0; nx < colGroups.length; nx++) { cgInserters[nx] = colGroups[nx].getInserter(name, false, checkType); } this.finishWriter = finishWriter; this.partition = partition; sClosed = false; } catch (Exception e) { throw new IOException("BTInsert constructor failed :" + e.getMessage()); } finally { if (sClosed) { if (cgInserters != null) { for (int i = 0; i < cgInserters.length; ++i) { if (cgInserters[i] != null) { try { cgInserters[i].close(); } catch (Exception e) { // no-op } } } } } } } @Override public Schema getSchema() { return Writer.this.getSchema(); } @Override public void insert(BytesWritable key, Tuple row) throws IOException { if (sClosed) { throw new IOException("Inserter already closed"); } // break the input row into sub-tuples, then insert them into the // corresponding CGs int curTotal = 0; try { partition.insert(key, row); } catch (Exception e) { throw new IOException("insert failed : " + e.getMessage()); } for (int nx = 0; nx < colGroups.length; nx++) { Tuple subTuple = cgTuples[nx]; int numCols = subTuple.size(); cgInserters[nx].insert(key, subTuple); curTotal += numCols; } } @Override public void close() throws IOException { if (sClosed) return; sClosed = true; try { for (TableInserter ins : cgInserters) { ins.close(); } if (finishWriter) { BasicTable.Writer.this.finish(); } } finally { for (TableInserter ins : cgInserters) { try { ins.close(); } catch (Exception e) { // no-op } } if (finishWriter) { try { BasicTable.Writer.this.finish(); } catch (Exception e) { // no-op } } } } } } /** * Drop a Basic Table, all files consisting of the BasicTable will be removed. * * @param path * the path to the Basic Table. * @param conf * The configuration object. * @throws IOException */ public static void drop(Path path, Configuration conf) throws IOException { FileSystem fs = path.getFileSystem(conf); fs.delete(path, true); } static class SchemaFile { private Version version; String comparator; Schema logical; Schema[] physical; Partition partition; boolean sorted; SortInfo sortInfo = null; String storage; CGSchema[] cgschemas; // Array indicating if a physical schema is already dropped // It is probably better to create "CGProperties" class and // store multiple properties like name there. boolean[] cgDeletedFlags; // ctor for reading public SchemaFile(Path path, String[] deletedCGs, Configuration conf) throws IOException { readSchemaFile(path, deletedCGs, conf); } // ctor for reading from a job configuration object; we do not need a table path; // all information is held in the job configuration object. public SchemaFile(Configuration conf) throws IOException { String logicalStr = ZebraConf.getOutputSchema(conf); storage = ZebraConf.getOutputStorageHint(conf); String sortColumns = ZebraConf.getOutputSortColumns(conf) != null ? ZebraConf.getOutputSortColumns(conf) : ""; comparator = ZebraConf.getOutputComparator(conf) != null ? ZebraConf.getOutputComparator(conf) : ""; version = SCHEMA_VERSION; try { logical = new Schema(logicalStr); } catch (Exception e) { throw new IOException("Schema build failed :" + e.getMessage()); } try { partition = new Partition(logicalStr, storage, comparator, sortColumns); } catch (Exception e) { throw new IOException("Partition constructor failed :" + e.getMessage()); } cgschemas = partition.getCGSchemas(); physical = new Schema[cgschemas.length]; //cgDeletedFlags = new boolean[physical.length]; for (int nx = 0; nx < cgschemas.length; nx++) { physical[nx] = cgschemas[nx].getSchema(); } this.sortInfo = partition.getSortInfo(); this.sorted = partition.isSorted(); this.comparator = (this.sortInfo == null ? null : this.sortInfo.getComparator()); if (this.comparator == null) this.comparator = ""; String[] sortColumnStr = sortColumns.split(","); if (sortColumnStr.length > 0) { sortInfo = SortInfo.parse(SortInfo.toSortString(sortColumnStr), logical, comparator); } } public Schema[] getPhysicalSchema() { return physical; } // ctor for writing public SchemaFile(Path path, String btSchemaStr, String btStorageStr, String sortColumns, String btComparator, Configuration conf) throws IOException { storage = btStorageStr; try { partition = new Partition(btSchemaStr, btStorageStr, btComparator, sortColumns); } catch (Exception e) { throw new IOException("Partition constructor failed :" + e.getMessage()); } this.sortInfo = partition.getSortInfo(); this.sorted = partition.isSorted(); this.comparator = (this.sortInfo == null ? null : this.sortInfo.getComparator()); if (this.comparator == null) this.comparator = ""; logical = partition.getSchema(); cgschemas = partition.getCGSchemas(); physical = new Schema[cgschemas.length]; for (int nx = 0; nx < cgschemas.length; nx++) { physical[nx] = cgschemas[nx].getSchema(); } cgDeletedFlags = new boolean[physical.length]; version = SCHEMA_VERSION; // write out the schema createSchemaFile(path, conf); } public String getComparator() { return comparator; } public Partition getPartition() { return partition; } public boolean isSorted() { return sorted; } public SortInfo getSortInfo() { return sortInfo; } public Schema getLogical() { return logical; } public int getNumOfPhysicalSchemas() { return physical.length; } public Schema getPhysicalSchema(int nx) { return physical[nx]; } public String getName(int nx) { return cgschemas[nx].getName(); } public String getSerializer(int nx) { return cgschemas[nx].getSerializer(); } public String getCompressor(int nx) { return cgschemas[nx].getCompressor(); } /** * Returns the index for CG with the given name. -1 indicates that there is * no CG with the name. */ int getCGByName(String cgName) { for (int i = 0; i < physical.length; i++) { if (cgName.equals(getName(i))) { return i; } } return -1; } /** Returns if the CG at the given index is delete */ boolean isCGDeleted(int idx) { return cgDeletedFlags[idx]; } public String getOwner(int nx) { return cgschemas[nx].getOwner(); } public String getGroup(int nx) { return cgschemas[nx].getGroup(); } public short getPerm(int nx) { return cgschemas[nx].getPerm(); } /** * @return the string representation of the physical schema. */ public String getBTSchemaString() { return logical.toString(); } /** * @return the string representation of the storage hints */ public String getStorageString() { return storage; } private void createSchemaFile(Path path, Configuration conf) throws IOException { // TODO: overwrite existing schema file, or need a flag? FSDataOutputStream outSchema = path.getFileSystem(conf).create(makeSchemaFilePath(path), true); version.write(outSchema); WritableUtils.writeString(outSchema, comparator); WritableUtils.writeString(outSchema, logical.toString()); WritableUtils.writeString(outSchema, storage); WritableUtils.writeVInt(outSchema, physical.length); for (int nx = 0; nx < physical.length; nx++) { WritableUtils.writeString(outSchema, physical[nx].toString()); } WritableUtils.writeVInt(outSchema, sorted ? 1 : 0); WritableUtils.writeVInt(outSchema, sortInfo == null ? 0 : sortInfo.size()); if (sortInfo != null && sortInfo.size() > 0) { String[] sortedCols = sortInfo.getSortColumnNames(); for (int i = 0; i < sortInfo.size(); i++) { WritableUtils.writeString(outSchema, sortedCols[i]); } } outSchema.close(); } private void readSchemaFile(Path path, String[] deletedCGs, Configuration conf) throws IOException { Path pathSchema = makeSchemaFilePath(path); if (!path.getFileSystem(conf).exists(pathSchema)) { throw new IOException("BT Schema file doesn't exist: " + pathSchema); } // read schema file FSDataInputStream in = path.getFileSystem(conf).open(pathSchema); version = new Version(in); // verify compatibility against SCHEMA_VERSION if (!version.compatibleWith(SCHEMA_VERSION)) { new IOException( "Incompatible versions, expecting: " + SCHEMA_VERSION + "; found in file: " + version); } comparator = WritableUtils.readString(in); String logicalStr = WritableUtils.readString(in); try { logical = new Schema(logicalStr); } catch (Exception e) { ; throw new IOException("Schema build failed :" + e.getMessage()); } storage = WritableUtils.readString(in); try { partition = new Partition(logicalStr, storage, comparator); } catch (Exception e) { throw new IOException("Partition constructor failed :" + e.getMessage()); } cgschemas = partition.getCGSchemas(); int numCGs = WritableUtils.readVInt(in); physical = new Schema[numCGs]; cgDeletedFlags = new boolean[physical.length]; TableSchemaParser parser; String cgschemastr; try { for (int nx = 0; nx < numCGs; nx++) { cgschemastr = WritableUtils.readString(in); parser = new TableSchemaParser(new StringReader(cgschemastr)); physical[nx] = parser.RecordSchema(null); } } catch (Exception e) { throw new IOException("parser.RecordSchema failed :" + e.getMessage()); } sorted = WritableUtils.readVInt(in) == 1 ? true : false; if (deletedCGs == null) setCGDeletedFlags(path, conf); else { for (String deletedCG : deletedCGs) { for (int i = 0; i < cgschemas.length; i++) { if (cgschemas[i].getName().equals(deletedCG)) cgDeletedFlags[i] = true; } } } if (version.compareTo(new Version((short) 1, (short) 0)) > 0) { int numSortColumns = WritableUtils.readVInt(in); if (numSortColumns > 0) { String[] sortColumnStr = new String[numSortColumns]; for (int i = 0; i < numSortColumns; i++) { sortColumnStr[i] = WritableUtils.readString(in); } sortInfo = SortInfo.parse(SortInfo.toSortString(sortColumnStr), logical, comparator); } } in.close(); } private static int getNumCGs(Path path, Configuration conf) throws IOException { Path pathSchema = makeSchemaFilePath(path); if (!path.getFileSystem(conf).exists(pathSchema)) { throw new IOException("BT Schema file doesn't exist: " + pathSchema); } // read schema file FSDataInputStream in = path.getFileSystem(conf).open(pathSchema); Version version = new Version(in); // verify compatibility against SCHEMA_VERSION if (!version.compatibleWith(SCHEMA_VERSION)) { new IOException( "Incompatible versions, expecting: " + SCHEMA_VERSION + "; found in file: " + version); } // read comparator WritableUtils.readString(in); // read logicalStr WritableUtils.readString(in); // read storage WritableUtils.readString(in); int numCGs = WritableUtils.readVInt(in); in.close(); return numCGs; } private static Path makeSchemaFilePath(Path parent) { return new Path(parent, BT_SCHEMA_FILE); } /** * Sets cgDeletedFlags array by checking presense of * ".deleted-CGNAME" directory in the table top level * directory. */ void setCGDeletedFlags(Path path, Configuration conf) throws IOException { Set<String> deletedCGs = new HashSet<String>(); for (FileStatus file : path.getFileSystem(conf).listStatus(path)) { if (!file.isDir()) { String fname = file.getPath().getName(); if (fname.startsWith(DELETED_CG_PREFIX)) { deletedCGs.add(fname.substring(DELETED_CG_PREFIX.length())); } } } for (int i = 0; i < physical.length; i++) { cgDeletedFlags[i] = deletedCGs.contains(getName(i)); } } String getDeletedCGs() { StringBuilder sb = new StringBuilder(); // comma separated boolean first = true; for (int i = 0; i < physical.length; i++) { if (cgDeletedFlags[i]) { if (first) first = false; else { sb.append(DELETED_CG_SEPARATOR_PER_TABLE); } sb.append(getName(i)); } } return sb.toString(); } } static public void dumpInfo(String file, PrintStream out, Configuration conf) throws IOException { dumpInfo(file, out, conf, 0); } static public void dumpInfo(String file, PrintStream out, Configuration conf, int indent) throws IOException { IOutils.indent(out, indent); out.println("Basic Table : " + file); Path path = new Path(file); try { BasicTable.Reader reader = new BasicTable.Reader(path, conf); String schemaStr = reader.getBTSchemaString(); String storageStr = reader.getStorageString(); IOutils.indent(out, indent); out.printf("Schema : %s\n", schemaStr); IOutils.indent(out, indent); out.printf("Storage Information : %s\n", storageStr); SortInfo sortInfo = reader.getSortInfo(); if (sortInfo != null && sortInfo.size() > 0) { IOutils.indent(out, indent); String[] sortedCols = sortInfo.getSortColumnNames(); out.println("Sorted Columns :"); for (int nx = 0; nx < sortedCols.length; nx++) { if (nx > 0) out.printf(" , "); out.printf("%s", sortedCols[nx]); } out.printf("\n"); } IOutils.indent(out, indent); out.println("Column Groups within the Basic Table :"); for (int nx = 0; nx < reader.colGroups.length; nx++) { IOutils.indent(out, indent); out.printf("\nColumn Group [%d] :", nx); if (reader.colGroups[nx] != null) { ColumnGroup.dumpInfo(reader.colGroups[nx].path, out, conf, indent); } else { // print basic info for deleted column groups. out.printf("\nColum Group : DELETED"); out.printf("\nName : %s", reader.schemaFile.getName(nx)); out.printf("\nSchema : %s\n", reader.schemaFile.cgschemas[nx].getSchema().toString()); } } } catch (Exception e) { throw new IOException("BasicTable.Reader failed : " + e.getMessage()); } finally { // no-op } } public static void main(String[] args) { System.out.printf("BasicTable Dumper\n"); if (args.length == 0) { System.out.println("Usage: java ... org.apache.hadoop.zebra.io.BasicTable path [path ...]"); System.exit(0); } Configuration conf = new Configuration(); for (String file : args) { try { dumpInfo(file, System.out, conf); } catch (IOException e) { e.printStackTrace(System.err); } } } }