Example usage for org.apache.hadoop.fs FileSystem getDefaultUri

List of usage examples for org.apache.hadoop.fs FileSystem getDefaultUri

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.fs.*
  5. FileSystem
  6. getDefaultUri

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem getDefaultUri.

Prototype

public static URI getDefaultUri(Configuration conf)

Source Link

Document

Get the default FileSystem URI from a configuration.

Usage

From source file:org.apache.ignite.internal.processors.hadoop.impl.delegate.HadoopBasicFileSystemFactoryDelegate.java

License:Apache License

/** {@inheritDoc} */
@Override//  w  w w. j  a va2 s . co  m
public void start() throws IgniteException {
    BasicHadoopFileSystemFactory proxy0 = (BasicHadoopFileSystemFactory) proxy;

    cfg = HadoopUtils.safeCreateConfiguration();

    if (proxy0.getConfigPaths() != null) {
        for (String cfgPath : proxy0.getConfigPaths()) {
            if (cfgPath == null)
                throw new NullPointerException(
                        "Configuration path cannot be null: " + Arrays.toString(proxy0.getConfigPaths()));
            else {
                URL url = U.resolveIgniteUrl(cfgPath);

                if (url == null) {
                    // If secConfPath is given, it should be resolvable:
                    throw new IgniteException("Failed to resolve secondary file system configuration path "
                            + "(ensure that it exists locally and you have read access to it): " + cfgPath);
                }

                cfg.addResource(url);
            }
        }
    }

    // If secondary fs URI is not given explicitly, try to get it from the configuration:
    if (proxy0.getUri() == null)
        fullUri = FileSystem.getDefaultUri(cfg);
    else {
        try {
            fullUri = new URI(proxy0.getUri());
        } catch (URISyntaxException ignored) {
            throw new IgniteException("Failed to resolve secondary file system URI: " + proxy0.getUri());
        }
    }

    String strWorkDir = fullUri.getPath();

    if (!"/".equals(strWorkDir))
        workDir = new Path(strWorkDir);

    usrNameMapper = proxy0.getUserNameMapper();

    if (usrNameMapper != null && usrNameMapper instanceof LifecycleAware)
        ((LifecycleAware) usrNameMapper).start();
}

From source file:org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider.java

License:Apache License

/**
 * Creates new provider with given config parameters. The configuration URL is optional. The filesystem URI must be
 * specified either explicitly or in the configuration provided.
 *
 * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified as "fs.defaultFS"
 * property in the provided configuration.
 * @param secConfPath the secondary Fs path (file path on the local file system, optional).
 * See {@link IgniteUtils#resolveIgniteUrl(String)} on how the path resolved.
 * @param userName User name.//from w  w w  . j av  a 2 s  .  c  o  m
 * @throws IOException
 */
public SecondaryFileSystemProvider(final @Nullable String secUri, final @Nullable String secConfPath,
        @Nullable String userName) throws IOException {
    this.userName = userName;

    if (secConfPath != null) {
        URL url = U.resolveIgniteUrl(secConfPath);

        if (url == null) {
            // If secConfPath is given, it should be resolvable:
            throw new IllegalArgumentException(
                    "Failed to resolve secondary file system " + "configuration path: " + secConfPath);
        }

        cfg.addResource(url);
    }

    // if secondary fs URI is not given explicitly, try to get it from the configuration:
    if (secUri == null)
        uri = FileSystem.getDefaultUri(cfg);
    else {
        try {
            uri = new URI(secUri);
        } catch (URISyntaxException use) {
            throw new IOException("Failed to resolve secondary file system URI: " + secUri);
        }
    }

    // Disable caching:
    String prop = String.format("fs.%s.impl.disable.cache", uri.getScheme());

    cfg.setBoolean(prop, true);
}

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

public static FileSystem getDefaultFileSystem() throws IOException {
    Path path = new Path(FileSystem.getDefaultUri(CONF));
    FileSystem fs = path.getFileSystem(CONF);
    return fs;//  w  w  w  . j a v  a 2  s.c om
}

From source file:org.apache.impala.common.FileSystemUtil.java

License:Apache License

/**
 * Fully-qualifies the given path based on the FileSystem configuration.
 *//*from  w ww  . j a v  a 2s .  c o m*/
public static Path createFullyQualifiedPath(Path location) {
    URI defaultUri = FileSystem.getDefaultUri(CONF);
    URI locationUri = location.toUri();
    // Use the default URI only if location has no scheme or it has the same scheme as
    // the default URI.  Otherwise, Path.makeQualified() will incorrectly use the
    // authority from the default URI even though the schemes don't match.  See HDFS-7031.
    if (locationUri.getScheme() == null || locationUri.getScheme().equalsIgnoreCase(defaultUri.getScheme())) {
        return location.makeQualified(defaultUri, location);
    }
    // Already qualified (has scheme).
    return location;
}

From source file:org.apache.nifi.processors.hadoop.AbstractHadoopProcessor.java

License:Apache License

HdfsResources resetHDFSResources(String configResources, ProcessContext context) throws IOException {
    Configuration config = new ExtendedConfiguration(getLogger());
    config.setClassLoader(Thread.currentThread().getContextClassLoader());

    getConfigurationFromResources(config, configResources);

    // give sub-classes a chance to process configuration
    preProcessConfiguration(config, context);

    // first check for timeout on HDFS connection, because FileSystem has a hard coded 15 minute timeout
    checkHdfsUriForTimeout(config);//  ww  w. ja v  a 2  s  .  co  m

    // disable caching of Configuration and FileSystem objects, else we cannot reconfigure the processor without a complete
    // restart
    String disableCacheName = String.format("fs.%s.impl.disable.cache",
            FileSystem.getDefaultUri(config).getScheme());
    config.set(disableCacheName, "true");

    // If kerberos is enabled, create the file system as the kerberos principal
    // -- use RESOURCE_LOCK to guarantee UserGroupInformation is accessed by only a single thread at at time
    FileSystem fs;
    UserGroupInformation ugi;
    synchronized (RESOURCES_LOCK) {
        if (SecurityUtil.isSecurityEnabled(config)) {
            String principal = context.getProperty(kerberosProperties.getKerberosPrincipal())
                    .evaluateAttributeExpressions().getValue();
            String keyTab = context.getProperty(kerberosProperties.getKerberosKeytab())
                    .evaluateAttributeExpressions().getValue();
            ugi = SecurityUtil.loginKerberos(config, principal, keyTab);
            fs = getFileSystemAsUser(config, ugi);
        } else {
            config.set("ipc.client.fallback-to-simple-auth-allowed", "true");
            config.set("hadoop.security.authentication", "simple");
            ugi = SecurityUtil.loginSimple(config);
            fs = getFileSystemAsUser(config, ugi);
        }
    }
    getLogger().debug("resetHDFSResources UGI {}", new Object[] { ugi });

    final Path workingDir = fs.getWorkingDirectory();
    getLogger().info(
            "Initialized a new HDFS File System with working dir: {} default block size: {} default replication: {} config: {}",
            new Object[] { workingDir, fs.getDefaultBlockSize(workingDir), fs.getDefaultReplication(workingDir),
                    config.toString() });

    return new HdfsResources(config, fs, ugi);
}

From source file:org.apache.pulsar.io.hdfs.AbstractHdfsConnector.java

License:Apache License

protected HdfsResources resetHDFSResources(HdfsSinkConfig hdfsSinkConfig) throws IOException {
    Configuration config = new ExtendedConfiguration();
    config.setClassLoader(Thread.currentThread().getContextClassLoader());

    getConfig(config, connectorConfig.getHdfsConfigResources());

    // first check for timeout on HDFS connection, because FileSystem has a hard coded 15 minute timeout
    checkHdfsUriForTimeout(config);//from   w ww .j a  v a 2 s.  com

    /* Disable caching of Configuration and FileSystem objects, else we cannot reconfigure
     * the processor without a complete restart
     */
    String disableCacheName = String.format("fs.%s.impl.disable.cache",
            FileSystem.getDefaultUri(config).getScheme());
    config.set(disableCacheName, "true");

    // If kerberos is enabled, create the file system as the kerberos principal
    // -- use RESOURCE_LOCK to guarantee UserGroupInformation is accessed by only a single thread at at time
    FileSystem fs;
    UserGroupInformation ugi;
    synchronized (RESOURCES_LOCK) {
        if (SecurityUtil.isSecurityEnabled(config)) {
            ugi = SecurityUtil.loginKerberos(config, connectorConfig.getKerberosUserPrincipal(),
                    connectorConfig.getKeytab());
            fs = getFileSystemAsUser(config, ugi);
        } else {
            config.set("ipc.client.fallback-to-simple-auth-allowed", "true");
            config.set("hadoop.security.authentication", "simple");
            ugi = SecurityUtil.loginSimple(config);
            fs = getFileSystemAsUser(config, ugi);
        }
    }
    return new HdfsResources(config, fs, ugi);
}

From source file:org.apache.sentry.binding.hive.authz.HiveAuthzBindingHookBase.java

License:Apache License

@VisibleForTesting
static public AccessURI parseURI(String uri, boolean isLocal) throws SemanticException {
    try {/*  ww  w. ja  v  a2  s.c o m*/
        HiveConf conf = SessionState.get().getConf();
        String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE);
        Path warehousePath = new Path(warehouseDir);

        // If warehousePath is an absolute path and a scheme is null and authority is null as well,
        // qualified it with default file system scheme and authority.
        if (warehousePath.isAbsoluteAndSchemeAuthorityNull()) {
            URI defaultUri = FileSystem.getDefaultUri(conf);
            warehousePath = warehousePath.makeQualified(defaultUri, warehousePath);
            warehouseDir = warehousePath.toUri().toString();
        }
        return new AccessURI(PathUtils.parseURI(warehouseDir, uri, isLocal));
    } catch (Exception e) {
        throw new SemanticException("Error parsing URI " + uri + ": " + e.getMessage(), e);
    }
}

From source file:org.apache.sentry.binding.hive.HiveAuthzBindingHookBase.java

License:Apache License

@VisibleForTesting
protected static AccessURI parseURI(String uri, boolean isLocal) throws SemanticException {
    try {/* w w  w  .jav  a2s  . c om*/
        HiveConf conf = SessionState.get().getConf();
        String warehouseDir = conf.getVar(ConfVars.METASTOREWAREHOUSE);
        Path warehousePath = new Path(warehouseDir);

        // If warehousePath is an absolute path and a scheme is null and authority is null as well,
        // qualified it with default file system scheme and authority.
        if (warehousePath.isAbsoluteAndSchemeAuthorityNull()) {
            URI defaultUri = FileSystem.getDefaultUri(conf);
            warehousePath = warehousePath.makeQualified(defaultUri, warehousePath);
            warehouseDir = warehousePath.toUri().toString();
        }
        return new AccessURI(PathUtils.parseURI(warehouseDir, uri, isLocal));
    } catch (Exception e) {
        throw new SemanticException("Error parsing URI " + uri + ": " + e.getMessage(), e);
    }
}

From source file:org.apache.sentry.core.common.utils.PathUtils.java

License:Apache License

/**
 * Make fully qualified URI if Scheme and/or Authority is missing,
 * based on the default file system Scheme and Authority.
 * Notes://ww w .  j  a  va2 s.c o  m
 * a) input URI path must be absolute; otherwise return null.
 * b) Path.makeQualified() provides no assurance that the
 *    default file system Scheme and Authority values are not null.
 *
 * @param uriName The Uri name.
 * @return Returns the fully qualified URI or null if URI path is not absolute.
 * @throws IOException
 */
private static URI makeFullQualifiedURI(String uriName) throws IOException {
    Path uriPath = new Path(uriName);
    if (isNormalized(uriName) && uriPath.isUriPathAbsolute()) {
        // add scheme and/or authority if either is missing
        if ((uriPath.toUri().getScheme() == null || uriPath.toUri().getAuthority() == null)) {
            URI defaultUri = FileSystem.getDefaultUri(CONF);
            uriPath = uriPath.makeQualified(defaultUri, uriPath);
        }
        return uriPath.toUri();
    } else { // relative URI path is unacceptable
        return null;
    }
}

From source file:org.apache.sentry.tests.e2e.hive.AbstractTestWithStaticConfiguration.java

License:Apache License

@BeforeClass
public static void setupTestStaticConfiguration() throws Exception {
    LOGGER.info("AbstractTestWithStaticConfiguration setupTestStaticConfiguration");
    properties = Maps.newHashMap();//  w ww. jav  a  2  s . c  om
    if (!policyOnHdfs) {
        policyOnHdfs = Boolean.valueOf(System.getProperty("sentry.e2etest.policyonhdfs", "false"));
    }
    if (testServerType != null) {
        properties.put("sentry.e2etest.hiveServer2Type", testServerType);
    }
    baseDir = Files.createTempDir();
    LOGGER.info("BaseDir = " + baseDir);
    logDir = assertCreateDir(new File(baseDir, "log"));
    confDir = assertCreateDir(new File(baseDir, "etc"));
    dataDir = assertCreateDir(new File(baseDir, "data"));
    policyFileLocation = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME);

    dfsType = System.getProperty(TestFSContants.SENTRY_E2E_TEST_DFS_TYPE,
            DFSFactory.DFSType.MiniDFS.toString());
    dfs = DFSFactory.create(dfsType, baseDir, testServerType, enableHDFSAcls);
    fileSystem = dfs.getFileSystem();

    PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMIN1)
            .setUserGroupMapping(StaticUserGroup.getStaticMapping());
    policyFile.write(policyFileLocation);

    String policyURI;
    if (policyOnHdfs) {
        String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
        LOGGER.info("dfsUri " + dfsUri);
        policyURI = dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
        policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME;
    } else {
        policyURI = policyFileLocation.getPath();
    }

    if (enableAuthorizingObjectStore) {
        properties.put(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
                "org.apache.sentry.binding.metastore.AuthorizingObjectStore");
    } else {
        properties.put(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname,
                ConfVars.METASTORE_RAW_STORE_IMPL.defaultStrVal);
    }

    if (enableFilter) {
        properties.put(ConfVars.METASTORE_FILTER_HOOK.varname,
                "org.apache.sentry.binding.metastore.SentryMetaStoreFilterHook");
    } else {
        properties.put(ConfVars.METASTORE_FILTER_HOOK.varname, ConfVars.METASTORE_FILTER_HOOK.defaultStrVal);
    }

    if (enableAuthorizeReadMetaData) {
        properties.put("senry.metastore.read.authorization.enabled", "true");
    } else {
        properties.put("senry.metastore.read.authorization.enabled", "false");
    }

    if (enableHiveConcurrency) {
        properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true");
        properties.put(HiveConf.ConfVars.HIVE_TXN_MANAGER.varname,
                "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager");
        properties.put(HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname,
                "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
    }
    if (useSentryService && (!startSentry)) {
        configureHiveAndMetastoreForSentry();
        setupSentryService();
    }

    if (defaultFSOnHdfs) {
        properties.put("fs.defaultFS", fileSystem.getUri().toString());
    }

    hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem);
    hiveServer.start();

    createContext();

    // Create tmp as scratch dir if it doesn't exist
    Path tmpPath = new Path("/tmp");
    if (!fileSystem.exists(tmpPath)) {
        fileSystem.mkdirs(tmpPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL));
    }
}