Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.hdfs.protocol; import java.util.Comparator; import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; import org.apache.hadoop.classification.InterfaceAudience; /** * This class tracks changes in the layout version of HDFS. * * Layout version is changed for following reasons: * <ol> * <li>The layout of how namenode or datanode stores information * on disk changes.</li> * <li>A new operation code is added to the editlog.</li> * <li>Modification such as format of a record, content of a record * in editlog or fsimage.</li> * </ol> * <br> * <b>How to update layout version:<br></b> * When a change requires new layout version, please add an entry into * {@link Feature} with a short enum name, new layout version and description * of the change. Please see {@link Feature} for further details. * <br> */ @InterfaceAudience.Private public class LayoutVersion { /** * Version in which HDFS-2991 was fixed. This bug caused OP_ADD to * sometimes be skipped for append() calls. If we see such a case when * loading the edits, but the version is known to have that bug, we * workaround the issue. Otherwise we should consider it a corruption * and bail. */ public static final int BUGFIX_HDFS_2991_VERSION = -40; /** * The interface to be implemented by NameNode and DataNode layout features */ public interface LayoutFeature { public FeatureInfo getInfo(); } /** * Enums for features that change the layout version before rolling * upgrade is supported. * <br><br> * To add a new layout version: * <ul> * <li>Define a new enum constant with a short enum name, the new layout version * and description of the added feature.</li> * <li>When adding a layout version with an ancestor that is not same as * its immediate predecessor, use the constructor where a specific ancestor * can be passed. * </li> * </ul> */ public enum Feature implements LayoutFeature { NAMESPACE_QUOTA(-16, "Support for namespace quotas"), FILE_ACCESS_TIME(-17, "Support for access time on files"), DISKSPACE_QUOTA(-18, "Support for disk space quotas"), STICKY_BIT(-19, "Support for sticky bits"), APPEND_RBW_DIR(-20, "Datanode has \"rbw\" subdirectory for append"), ATOMIC_RENAME(-21, "Support for atomic rename"), CONCAT(-22, "Support for concat operation"), SYMLINKS(-23, "Support for symbolic links"), DELEGATION_TOKEN(-24, "Support for delegation tokens for security"), FSIMAGE_COMPRESSION( -25, "Support for fsimage compression"), FSIMAGE_CHECKSUM( -26, "Support checksum for fsimage"), REMOVE_REL13_DISK_LAYOUT_SUPPORT( -27, "Remove support for 0.13 disk layout"), EDITS_CHECKSUM( -28, "Support checksum for editlog"), UNUSED( -29, "Skipped version"), FSIMAGE_NAME_OPTIMIZATION( -30, "Store only last part of path in fsimage"), RESERVED_REL20_203( -31, -19, "Reserved for release 0.20.203", true, DELEGATION_TOKEN), RESERVED_REL20_204( -32, -31, "Reserved for release 0.20.204", true), RESERVED_REL22( -33, -27, "Reserved for release 0.22", true), RESERVED_REL23( -34, -30, "Reserved for release 0.23", true), FEDERATION( -35, "Support for namenode federation"), LEASE_REASSIGNMENT( -36, "Support for persisting lease holder reassignment"), STORED_TXIDS( -37, "Transaction IDs are stored in edits log and image files"), TXID_BASED_LAYOUT( -38, "File names in NN Storage are based on transaction IDs"), EDITLOG_OP_OPTIMIZATION( -39, "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"), OPTIMIZE_PERSIST_BLOCKS( -40, "Serialize block lists with delta-encoded variable length ints, " + "add OP_UPDATE_BLOCKS"), RESERVED_REL1_2_0( -41, -32, "Reserved for release 1.2.0", true, CONCAT), ADD_INODE_ID( -42, -40, "Assign a unique inode id for each inode", false), SNAPSHOT( -43, "Support for snapshot feature"), RESERVED_REL1_3_0( -44, -41, "Reserved for release 1.3.0", true, ADD_INODE_ID, SNAPSHOT, FSIMAGE_NAME_OPTIMIZATION), OPTIMIZE_SNAPSHOT_INODES( -45, -43, "Reduce snapshot inode memory footprint", false), SEQUENTIAL_BLOCK_ID( -46, "Allocate block IDs sequentially and store " + "block IDs in the edits log and image files"), EDITLOG_SUPPORT_RETRYCACHE( -47, "Record ClientId and CallId in editlog to " + "enable rebuilding retry cache in case of HA failover"), EDITLOG_ADD_BLOCK( -48, "Add new editlog that only records allocation of " + "the new block instead of the entire block list"), ADD_DATANODE_AND_STORAGE_UUIDS( -49, "Replace StorageID with DatanodeUuid." + " Use distinct StorageUuid per storage directory."), ADD_LAYOUT_FLAGS( -50, "Add support for layout flags."), CACHING( -51, "Support for cache pools and path-based caching"), // Hadoop 2.4.0 PROTOBUF_FORMAT(-52, "Use protobuf to serialize FSImage"), EXTENDED_ACL(-53, "Extended ACL"), RESERVED_REL2_4_0(-54, -51, "Reserved for release 2.4.0", true, PROTOBUF_FORMAT, EXTENDED_ACL); private final FeatureInfo info; /** * Feature that is added at layout version {@code lv} - 1. * @param lv new layout version with the addition of this feature * @param description description of the feature */ Feature(final int lv, final String description) { this(lv, lv + 1, description, false); } /** * Feature that is added at layout version {@code ancestoryLV}. * @param lv new layout version with the addition of this feature * @param ancestorLV layout version from which the new lv is derived from. * @param description description of the feature * @param reserved true when this is a layout version reserved for previous * version * @param features set of features that are to be enabled for this version */ Feature(final int lv, final int ancestorLV, final String description, boolean reserved, Feature... features) { info = new FeatureInfo(lv, ancestorLV, description, reserved, features); } @Override public FeatureInfo getInfo() { return info; } } /** Feature information. */ public static class FeatureInfo { private final int lv; private final int ancestorLV; private final Integer minCompatLV; private final String description; private final boolean reserved; private final LayoutFeature[] specialFeatures; public FeatureInfo(final int lv, final int ancestorLV, final String description, boolean reserved, LayoutFeature... specialFeatures) { this(lv, ancestorLV, null, description, reserved, specialFeatures); } public FeatureInfo(final int lv, final int ancestorLV, Integer minCompatLV, final String description, boolean reserved, LayoutFeature... specialFeatures) { this.lv = lv; this.ancestorLV = ancestorLV; this.minCompatLV = minCompatLV; this.description = description; this.reserved = reserved; this.specialFeatures = specialFeatures; } /** * Accessor method for feature layout version * @return int lv value */ public int getLayoutVersion() { return lv; } /** * Accessor method for feature ancestor layout version * @return int ancestor LV value */ public int getAncestorLayoutVersion() { return ancestorLV; } /** * Accessor method for feature minimum compatible layout version. If the * feature does not define a minimum compatible layout version, then this * method returns the feature's own layout version. This would indicate * that the feature cannot provide compatibility with any prior layout * version. * * @return int minimum compatible LV value */ public int getMinimumCompatibleLayoutVersion() { return minCompatLV != null ? minCompatLV : lv; } /** * Accessor method for feature description * @return String feature description */ public String getDescription() { return description; } public boolean isReservedForOldRelease() { return reserved; } public LayoutFeature[] getSpecialFeatures() { return specialFeatures; } } static class LayoutFeatureComparator implements Comparator<LayoutFeature> { @Override public int compare(LayoutFeature arg0, LayoutFeature arg1) { return arg0.getInfo().getLayoutVersion() - arg1.getInfo().getLayoutVersion(); } } public static void updateMap(Map<Integer, SortedSet<LayoutFeature>> map, LayoutFeature[] features) { // Go through all the enum constants and build a map of // LayoutVersion <-> Set of all supported features in that LayoutVersion SortedSet<LayoutFeature> existingFeatures = new TreeSet<LayoutFeature>(new LayoutFeatureComparator()); for (SortedSet<LayoutFeature> s : map.values()) { existingFeatures.addAll(s); } LayoutFeature prevF = existingFeatures.isEmpty() ? null : existingFeatures.first(); for (LayoutFeature f : features) { final FeatureInfo info = f.getInfo(); int minCompatLV = info.getMinimumCompatibleLayoutVersion(); if (prevF != null && minCompatLV > prevF.getInfo().getMinimumCompatibleLayoutVersion()) { throw new AssertionError( String.format("Features must be listed in order of minimum compatible layout " + "version. Check features %s and %s.", prevF, f)); } prevF = f; SortedSet<LayoutFeature> ancestorSet = map.get(info.getAncestorLayoutVersion()); if (ancestorSet == null) { // Empty set ancestorSet = new TreeSet<LayoutFeature>(new LayoutFeatureComparator()); map.put(info.getAncestorLayoutVersion(), ancestorSet); } SortedSet<LayoutFeature> featureSet = new TreeSet<LayoutFeature>(ancestorSet); if (info.getSpecialFeatures() != null) { for (LayoutFeature specialFeature : info.getSpecialFeatures()) { featureSet.add(specialFeature); } } featureSet.add(f); map.put(info.getLayoutVersion(), featureSet); } } /** * Gets formatted string that describes {@link LayoutVersion} information. */ public String getString(Map<Integer, SortedSet<LayoutFeature>> map, LayoutFeature[] values) { final StringBuilder buf = new StringBuilder(); buf.append("Feature List:\n"); for (LayoutFeature f : values) { final FeatureInfo info = f.getInfo(); buf.append(f).append(" introduced in layout version ").append(info.getLayoutVersion()).append(" (") .append(info.getDescription()).append(")\n"); } buf.append("\n\nLayoutVersion and supported features:\n"); for (LayoutFeature f : values) { final FeatureInfo info = f.getInfo(); buf.append(info.getLayoutVersion()).append(": ").append(map.get(info.getLayoutVersion())).append("\n"); } return buf.toString(); } /** * Returns true if a given feature is supported in the given layout version * @param map layout feature map * @param f Feature * @param lv LayoutVersion * @return true if {@code f} is supported in layout version {@code lv} */ public static boolean supports(Map<Integer, SortedSet<LayoutFeature>> map, final LayoutFeature f, final int lv) { final SortedSet<LayoutFeature> set = map.get(lv); return set != null && set.contains(f); } /** * Get the current layout version */ public static int getCurrentLayoutVersion(LayoutFeature[] features) { return getLastNonReservedFeature(features).getInfo().getLayoutVersion(); } /** * Gets the minimum compatible layout version. * * @param features all features to check * @return minimum compatible layout version */ public static int getMinimumCompatibleLayoutVersion(LayoutFeature[] features) { return getLastNonReservedFeature(features).getInfo().getMinimumCompatibleLayoutVersion(); } static LayoutFeature getLastNonReservedFeature(LayoutFeature[] features) { for (int i = features.length - 1; i >= 0; i--) { final FeatureInfo info = features[i].getInfo(); if (!info.isReservedForOldRelease()) { return features[i]; } } throw new AssertionError("All layout versions are reserved."); } }