Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ignite.internal.processors.hadoop.impl.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.ignite.IgniteException; import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem; import org.apache.ignite.internal.util.GridStringBuilder; import org.apache.ignite.internal.util.typedef.F; import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.net.URI; /** * File system cache utility methods used by Map-Reduce tasks and jobs. */ public class HadoopFileSystemCacheUtils { /** * A common static factory method. Creates new HadoopLazyConcurrentMap. * @return a new HadoopLazyConcurrentMap. */ public static HadoopLazyConcurrentMap<FsCacheKey, FileSystem> createHadoopLazyConcurrentMap() { return new HadoopLazyConcurrentMap<>(new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() { @Override public FileSystem createValue(FsCacheKey key) throws IOException { try { assert key != null; // Explicitly disable FileSystem caching: URI uri = key.uri(); String scheme = uri.getScheme(); // Copy the configuration to avoid altering the external object. Configuration cfg = new Configuration(key.configuration()); String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(scheme); cfg.setBoolean(prop, true); return FileSystem.get(uri, cfg, key.user()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IOException("Failed to create file system due to interrupt.", e); } } }); } /** * Gets non-null user name as per the Hadoop viewpoint. * @param cfg the Hadoop job configuration, may be null. * @return the user name, never null. */ private static String getMrHadoopUser(Configuration cfg) throws IOException { String user = cfg.get(MRJobConfig.USER_NAME); if (user == null) user = IgniteHadoopFileSystem.getFsHadoopUser(); return user; } /** * Common method to get the V1 file system in MapRed engine. * It gets the filesystem for the user specified in the * configuration with {@link MRJobConfig#USER_NAME} property. * The file systems are created and cached in the given map upon first request. * * @param uri The file system uri. * @param cfg The configuration. * @param map The caching map. * @return The file system. * @throws IOException On error. */ public static FileSystem fileSystemForMrUserWithCaching(@Nullable URI uri, Configuration cfg, HadoopLazyConcurrentMap<FsCacheKey, FileSystem> map) throws IOException { assert map != null; assert cfg != null; final String usr = getMrHadoopUser(cfg); assert usr != null; if (uri == null) uri = FileSystem.getDefaultUri(cfg); final FileSystem fs; try { final FsCacheKey key = new FsCacheKey(uri, usr, cfg); fs = map.getOrCreate(key); } catch (IgniteException ie) { throw new IOException(ie); } assert fs != null; assert !(fs instanceof IgniteHadoopFileSystem) || F.eq(usr, ((IgniteHadoopFileSystem) fs).user()); return fs; } /** * Takes Fs URI using logic similar to that used in FileSystem#get(1,2,3). * @param uri0 The uri. * @param cfg The cfg. * @return Correct URI. */ private static URI fixUri(URI uri0, Configuration cfg) { if (uri0 == null) return FileSystem.getDefaultUri(cfg); String scheme = uri0.getScheme(); String authority = uri0.getAuthority(); if (authority == null) { URI dfltUri = FileSystem.getDefaultUri(cfg); if (scheme == null || (scheme.equals(dfltUri.getScheme()) && dfltUri.getAuthority() != null)) return dfltUri; } return uri0; } /** * Note that configuration is not a part of the key. * It is used solely to initialize the first instance * that is created for the key. */ public static final class FsCacheKey { /** */ private final URI uri; /** */ private final String usr; /** */ private final String equalityKey; /** */ private final Configuration cfg; /** * Constructor */ public FsCacheKey(URI uri, String usr, Configuration cfg) { assert uri != null; assert usr != null; assert cfg != null; this.uri = fixUri(uri, cfg); this.usr = usr; this.cfg = cfg; this.equalityKey = createEqualityKey(); } /** * Creates String key used for equality and hashing. */ private String createEqualityKey() { GridStringBuilder sb = new GridStringBuilder("(").a(usr).a(")@"); if (uri.getScheme() != null) sb.a(uri.getScheme().toLowerCase()); sb.a("://"); if (uri.getAuthority() != null) sb.a(uri.getAuthority().toLowerCase()); return sb.toString(); } /** * The URI. */ public URI uri() { return uri; } /** * The User. */ public String user() { return usr; } /** * The Configuration. */ public Configuration configuration() { return cfg; } /** {@inheritDoc} */ @SuppressWarnings("SimplifiableIfStatement") @Override public boolean equals(Object obj) { if (obj == this) return true; if (obj == null || getClass() != obj.getClass()) return false; return equalityKey.equals(((FsCacheKey) obj).equalityKey); } /** {@inheritDoc} */ @Override public int hashCode() { return equalityKey.hashCode(); } /** {@inheritDoc} */ @Override public String toString() { return equalityKey; } } }