Java tutorial
/** * Copyright 2011-2017 Asakusa Framework Team. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.asakusafw.runtime.directio.hadoop; import java.io.IOException; import java.text.MessageFormat; import java.util.HashMap; import java.util.Map; import java.util.TreeSet; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import com.asakusafw.runtime.directio.DirectDataSourceProfile; import com.asakusafw.runtime.directio.FragmentableDataFormat; /** * A structured profile for {@link HadoopDataSource}. * @since 0.2.5 * @version 0.9.0 */ public class HadoopDataSourceProfile { static final Log LOG = LogFactory.getLog(HadoopDataSourceProfile.class); // fs.staging private static final String ROOT_REPRESENTATION = "/"; //$NON-NLS-1$ /** * The property key name for {@link #getFileSystemPath()}. * Default is {@link FileSystem#getWorkingDirectory()}. */ public static final String KEY_PATH = "fs.path"; //$NON-NLS-1$ /** * The property key name for {@link #getTemporaryFileSystemPath()}. */ public static final String KEY_TEMP = "fs.tempdir"; //$NON-NLS-1$ /** * The property key name for {@link #isOutputStaging()}. */ public static final String KEY_OUTPUT_STAGING = "output.staging"; //$NON-NLS-1$ /** * The property key name for {@link #isOutputStreaming()}. */ public static final String KEY_OUTPUT_STREAMING = "output.streaming"; //$NON-NLS-1$ /** * The property key name for {@link #getMinimumFragmentSize(FragmentableDataFormat)}. */ public static final String KEY_MIN_FRAGMENT = "fragment.min"; //$NON-NLS-1$ /** * The property key name for {@link #getPreferredFragmentSize(FragmentableDataFormat)}. */ public static final String KEY_PREF_FRAGMENT = "fragment.pref"; //$NON-NLS-1$ /** * The property key name for {@link #isSplitBlocks()}. */ public static final String KEY_SPLIT_BLOCKS = "block.split"; //$NON-NLS-1$ /** * The property key name for {@link #isCombineBlocks()}. */ public static final String KEY_COMBINE_BLOCKS = "block.combine"; //$NON-NLS-1$ /** * The property key name for {@link #getKeepAliveInterval()}. * @since 0.2.6 */ public static final String KEY_KEEPALIVE_INTERVAL = "keepalive.interval"; //$NON-NLS-1$ /** * The property key name of number of threads for moving files in roll-forward operation. * @since 0.9.0 */ public static final String KEY_ROLLFORWARD_THREADS = "threads.commit"; //$NON-NLS-1$ private static final String DEFAULT_TEMP_SUFFIX = "_directio_temp"; //$NON-NLS-1$ private static final boolean DEFAULT_OUTPUT_STAGING = true; private static final boolean DEFAULT_OUTPUT_STREAMING = true; private static final long DEFAULT_MIN_FRAGMENT = 16 * 1024 * 1024; private static final long DEFAULT_PREF_FRAGMENT = 64 * 1024 * 1024; private static final boolean DEFAULT_SPLIT_BLOCKS = true; private static final boolean DEFAULT_COMBINE_BLOCKS = true; private static final long DEFAULT_KEEPALIVE_INTERVAL = 0; private static final int DEFAULT_ROLLFORWARD_THREADS = 1; private final String id; private final String contextPath; private final Path fileSystemPath; private final Path temporaryPath; private boolean outputStaging = DEFAULT_OUTPUT_STAGING; private boolean outputStreaming = DEFAULT_OUTPUT_STREAMING; private long minimumFragmentSize = DEFAULT_MIN_FRAGMENT; private long preferredFragmentSize = DEFAULT_PREF_FRAGMENT; private boolean splitBlocks = DEFAULT_SPLIT_BLOCKS; private boolean combineBlocks = DEFAULT_COMBINE_BLOCKS; private long keepAliveInterval = DEFAULT_KEEPALIVE_INTERVAL; private int rollforwardThreads = DEFAULT_ROLLFORWARD_THREADS; private final FileSystem fileSystem; private final LocalFileSystem localFileSystem; /** * Creates a new instance. * @param conf the current configuration * @param id the ID of this datasource * @param contextPath the logical context path * @param fileSystemPath the mapping target path * @param temporaryPath the temporary root path * @throws IOException if failed to create profile * @throws IllegalArgumentException if some parameters were {@code null} */ public HadoopDataSourceProfile(Configuration conf, String id, String contextPath, Path fileSystemPath, Path temporaryPath) throws IOException { this.id = id; this.contextPath = contextPath; this.fileSystemPath = fileSystemPath; this.temporaryPath = temporaryPath; this.fileSystem = fileSystemPath.getFileSystem(conf); this.localFileSystem = FileSystem.getLocal(conf); } /** * Return the ID of this datasource. * @return the ID */ public String getId() { return id; } /** * Returns the logical context path. * @return the logical context path */ public String getContextPath() { return contextPath; } /** * Returns the mapping target path. * @return the mapping target path */ public Path getFileSystemPath() { return fileSystemPath; } /** * Returns the temporary root path. * @return the temporary root path */ public Path getTemporaryFileSystemPath() { return temporaryPath; } /** * Returns the file system for the this datastore. * @return the file system object */ public FileSystem getFileSystem() { return fileSystem; } /** * Returns the local file system for the this datastore. * @return the local file system object */ public LocalFileSystem getLocalFileSystem() { return localFileSystem; } /** * Returns the minimum fragment size. * @param format target format * @return the minimum fragment size, or {@code < 0} if fragmentation is restricted * @throws IOException if failed to compute size by I/O error * @throws InterruptedException if interrupted * @throws IllegalArgumentException if some parameters were {@code null} */ public long getMinimumFragmentSize(FragmentableDataFormat<?> format) throws IOException, InterruptedException { if (format == null) { throw new IllegalArgumentException("format must not be null"); //$NON-NLS-1$ } long formatMin = format.getMinimumFragmentSize(); long totalMin = Math.min(formatMin, minimumFragmentSize); if (totalMin <= 0) { return -1; } return totalMin; } /** * Returns the minimum fragment size. * @return the minimum fragment size, or {@code < 0} if fragmentation is restricted */ public long getMinimumFragmentSize() { return minimumFragmentSize <= 0 ? -1 : minimumFragmentSize; } /** * Configures the minimum fragment size in bytes. * @param size the size, or {@code <= 0} to restrict fragmentation */ public void setMinimumFragmentSize(long size) { if (size <= 0) { this.minimumFragmentSize = -1; } this.minimumFragmentSize = size; } /** * Returns the preferred fragment size. * @param format target format * @return the preferred fragment size * @throws IOException if failed to compute size by I/O error * @throws InterruptedException if interrupted * @throws IllegalArgumentException if some parameters were {@code null} */ public long getPreferredFragmentSize(FragmentableDataFormat<?> format) throws IOException, InterruptedException { if (format == null) { throw new IllegalArgumentException("format must not be null"); //$NON-NLS-1$ } long min = getMinimumFragmentSize(format); if (min <= 0) { return -1; } long formatPref = format.getPreferredFragmentSize(); if (formatPref > 0) { return Math.max(formatPref, min); } return Math.max(preferredFragmentSize, min); } /** * Returns the preferred fragment size. * @return the preferred fragment size */ public long getPreferredFragmentSize() { long min = getMinimumFragmentSize(); if (min <= 0) { return -1; } return preferredFragmentSize <= 0 ? -1 : preferredFragmentSize; } /** * Configures the preferred fragment size in bytes. * @param size the size */ public void setPreferredFragmentSize(long size) { this.preferredFragmentSize = Math.max(size, 1); } /** * Returns whether split DFS block into multiple splits for optimization. * @return the {@code true} to split, otherwise {@code false} */ public boolean isSplitBlocks() { return splitBlocks; } /** * Sets whether splits blocks for optimization. * @param split {@code true} to split, otherwise {@code false} */ public void setSplitBlocks(boolean split) { this.splitBlocks = split; } /** * Returns whether combines multiple blocks into a fragment for optimization. * @return the {@code true} to combine, otherwise {@code false} */ public boolean isCombineBlocks() { return combineBlocks; } /** * Sets whether combines blocks for optimization. * @param combine {@code true} to combine, otherwise {@code false} */ public void setCombineBlocks(boolean combine) { this.combineBlocks = combine; } /** * Returns whether output staging is required. * @return {@code true} to required, otherwise {@code false}. */ public boolean isOutputStaging() { return outputStaging; } /** * Sets whether output staging is required. * @param required {@code true} to required, otherwise {@code false} */ public void setOutputStaging(boolean required) { this.outputStaging = required; } /** * Returns whether output streaming is required. * @return {@code true} to required, otherwise {@code false}. */ public boolean isOutputStreaming() { return outputStreaming; } /** * Sets whether output streaming is required. * @param required {@code true} to required, otherwise {@code false} */ public void setOutputStreaming(boolean required) { this.outputStreaming = required; } /** * Returns keep-alive interval. * @return keep-alive interval in ms, or {@code 0} if keep-alive is disabled * @since 0.2.6 */ public long getKeepAliveInterval() { return keepAliveInterval; } /** * Sets keep-alive interval. * @param interval keep-alive interval in ms, or {@code 0} to disable keep-alive * @since 0.2.6 */ public void setKeepAliveInterval(long interval) { this.keepAliveInterval = interval; } /** * Returns the number of threads to move staged files to committed area. * @return the number of threads * @since 0.9.0 */ public int getRollforwardThreads() { return rollforwardThreads; } /** * Sets the number of threads to move staged files to committed area. * @param threads the number of threads * @since 0.9.0 */ public void setRollforwardThreads(int threads) { this.rollforwardThreads = threads; } @Override public String toString() { StringBuilder builder = new StringBuilder(); builder.append("HadoopDataSourceProfile [id="); //$NON-NLS-1$ builder.append(id); builder.append(", contextPath="); //$NON-NLS-1$ builder.append(contextPath); builder.append(", fileSystemPath="); //$NON-NLS-1$ builder.append(fileSystemPath); builder.append(", temporaryPath="); //$NON-NLS-1$ builder.append(temporaryPath); builder.append(", outputStaging="); //$NON-NLS-1$ builder.append(outputStaging); builder.append(", outputStreaming="); //$NON-NLS-1$ builder.append(outputStreaming); builder.append(", minimumFragmentSize="); //$NON-NLS-1$ builder.append(minimumFragmentSize); builder.append(", preferredFragmentSize="); //$NON-NLS-1$ builder.append(preferredFragmentSize); builder.append(", splitBlocks="); //$NON-NLS-1$ builder.append(splitBlocks); builder.append(", combineBlocks="); //$NON-NLS-1$ builder.append(combineBlocks); builder.append(", keepAliveInterval="); //$NON-NLS-1$ builder.append(keepAliveInterval); builder.append(", rollforwardThreads="); //$NON-NLS-1$ builder.append(rollforwardThreads); builder.append(", fileSystem="); //$NON-NLS-1$ builder.append(fileSystem); builder.append(", localFileSystem="); //$NON-NLS-1$ builder.append(localFileSystem); builder.append("]"); //$NON-NLS-1$ return builder.toString(); } /** * Converts the {@link DirectDataSourceProfile} into this profile. * @param profile target profile * @param conf Hadoop configuration * @return the converted profile * @throws IOException if failed to convert * @throws IllegalArgumentException if some parameters were {@code null} */ public static HadoopDataSourceProfile convert(DirectDataSourceProfile profile, Configuration conf) throws IOException { if (profile == null) { throw new IllegalArgumentException("profile must not be null"); //$NON-NLS-1$ } if (conf == null) { throw new IllegalArgumentException("conf must not be null"); //$NON-NLS-1$ } Map<String, String> attributes = new HashMap<>(profile.getAttributes()); Path fsPath = takeFsPath(profile, attributes, conf); if (fsPath == null) { throw new IOException(MessageFormat.format( "The directio configuration \"{0} ({1})\" does not have \"{2}\"", profile.getId(), profile.getPath().isEmpty() ? ROOT_REPRESENTATION : profile.getPath(), fqn(profile, KEY_PATH))); } Path tempPath = takeTempPath(profile, attributes, conf, fsPath); FileSystem fileSystem = fsPath.getFileSystem(conf); FileSystem tempFs = tempPath.getFileSystem(conf); if (getFsIdentity(fileSystem).equals(getFsIdentity(tempFs)) == false) { throw new IOException(MessageFormat.format( "The directio target and temporary path must be on same file system ({0}={1} <=> {2}={3})", fqn(profile, KEY_PATH), fsPath, fqn(profile, KEY_TEMP), tempPath)); } fsPath = fileSystem.makeQualified(fsPath); tempPath = fileSystem.makeQualified(tempPath); HadoopDataSourceProfile result = new HadoopDataSourceProfile(conf, profile.getId(), profile.getPath(), fsPath, tempPath); long minFragment = takeMinFragment(profile, attributes, conf); result.setMinimumFragmentSize(minFragment); long prefFragment = takePrefFragment(profile, attributes, conf); result.setPreferredFragmentSize(prefFragment); result.setOutputStaging(takeBoolean(profile, attributes, KEY_OUTPUT_STAGING, DEFAULT_OUTPUT_STAGING)); result.setOutputStreaming(takeBoolean(profile, attributes, KEY_OUTPUT_STREAMING, DEFAULT_OUTPUT_STREAMING)); result.setSplitBlocks(takeBoolean(profile, attributes, KEY_SPLIT_BLOCKS, DEFAULT_SPLIT_BLOCKS)); result.setCombineBlocks(takeBoolean(profile, attributes, KEY_COMBINE_BLOCKS, DEFAULT_COMBINE_BLOCKS)); result.setKeepAliveInterval( takePositive(profile, attributes, KEY_KEEPALIVE_INTERVAL, DEFAULT_KEEPALIVE_INTERVAL)); result.setRollforwardThreads( takePositive(profile, attributes, KEY_ROLLFORWARD_THREADS, DEFAULT_ROLLFORWARD_THREADS)); if (attributes.isEmpty() == false) { throw new IOException(MessageFormat.format("Unknown attributes in \"{0}\": {1}", profile.getId(), new TreeSet<>(attributes.keySet()))); } return result; } static String getFsIdentity(FileSystem fileSystem) { assert fileSystem != null; return fileSystem.getUri().toString(); } private static Object fqn(DirectDataSourceProfile profile, String key) { assert profile != null; assert key != null; return MessageFormat.format("{0}.{1}", //$NON-NLS-1$ profile.getId(), key); } private static Path takeFsPath(DirectDataSourceProfile profile, Map<String, String> attributes, Configuration conf) { assert conf != null; assert attributes != null; String fsPathString = attributes.remove(KEY_PATH); if (fsPathString != null) { return new Path(fsPathString); } return null; } private static Path takeTempPath(DirectDataSourceProfile profile, Map<String, String> attributes, Configuration conf, Path fsPath) { assert attributes != null; assert conf != null; assert fsPath != null; String tempPathString = attributes.remove(KEY_TEMP); Path tempPath; if (tempPathString != null) { tempPath = new Path(tempPathString); } else { tempPath = new Path(fsPath, DEFAULT_TEMP_SUFFIX); } return tempPath; } private static long takeMinFragment(DirectDataSourceProfile profile, Map<String, String> attributes, Configuration conf) throws IOException { assert profile != null; assert attributes != null; assert conf != null; String string = attributes.remove(KEY_MIN_FRAGMENT); if (string == null) { return DEFAULT_MIN_FRAGMENT; } try { long value = Long.parseLong(string); if (value == 0) { throw new IOException(MessageFormat.format("Minimum fragment size must not be zero: {0}", fqn(profile, KEY_MIN_FRAGMENT))); } return value; } catch (NumberFormatException e) { throw new IOException(MessageFormat.format("Minimum fragment size must be integer: {0}={1}", fqn(profile, KEY_MIN_FRAGMENT), string)); } } private static long takePrefFragment(DirectDataSourceProfile profile, Map<String, String> attributes, Configuration conf) throws IOException { assert profile != null; assert attributes != null; assert conf != null; String string = attributes.remove(KEY_PREF_FRAGMENT); if (string == null) { return DEFAULT_PREF_FRAGMENT; } try { long value = Long.parseLong(string); if (value <= 0) { throw new IOException(MessageFormat.format("Preferred fragment size must be > 0: {0}={1}", fqn(profile, KEY_PREF_FRAGMENT), string)); } return value; } catch (NumberFormatException e) { throw new IOException(MessageFormat.format("Preferred fragment size must be integer: {0}={1}", fqn(profile, KEY_PREF_FRAGMENT), string)); } } private static boolean takeBoolean(DirectDataSourceProfile profile, Map<String, String> attributes, String key, boolean defaultValue) throws IOException { assert profile != null; assert attributes != null; assert key != null; String string = attributes.remove(key); if (string == null) { return defaultValue; } if (string.equalsIgnoreCase("true")) { //$NON-NLS-1$ return true; } else if (string.equalsIgnoreCase("false")) { //$NON-NLS-1$ return false; } else { throw new IOException(MessageFormat.format("\"{0}\" must be boolean: {1}", fqn(profile, key), string)); } } private static int takePositive(DirectDataSourceProfile profile, Map<String, String> attributes, String key, int defaultValue) throws IOException { assert profile != null; assert attributes != null; assert key != null; String string = attributes.remove(key); if (string == null) { return defaultValue; } try { int result = Integer.parseInt(string.trim()); if (result < 0) { throw new IOException( MessageFormat.format("\"{0}\" must be positive integer: {1}", fqn(profile, key), string)); } return result; } catch (NumberFormatException e) { throw new IOException( MessageFormat.format("\"{0}\" must be positive integer: {1}", fqn(profile, key), string), e); } } private static long takePositive(DirectDataSourceProfile profile, Map<String, String> attributes, String key, long defaultValue) throws IOException { assert profile != null; assert attributes != null; assert key != null; String string = attributes.remove(key); if (string == null) { return defaultValue; } try { long result = Integer.parseInt(string.trim()); if (result < 0) { throw new IOException( MessageFormat.format("\"{0}\" must be positive integer: {1}", fqn(profile, key), string)); } return result; } catch (NumberFormatException e) { throw new IOException( MessageFormat.format("\"{0}\" must be positive integer: {1}", fqn(profile, key), string), e); } } }