List of usage examples for org.apache.hadoop.conf Configuration setClassLoader
public void setClassLoader(ClassLoader classLoader)
From source file:org.elasticsearch.repositories.hdfs.HdfsBlobStoreContainerTests.java
License:Apache License
@SuppressForbidden(reason = "lesser of two evils (the other being a bunch of JNI/classloader nightmares)") private FileContext createContext(URI uri) { // mirrors HdfsRepository.java behaviour Configuration cfg = new Configuration(true); cfg.setClassLoader(HdfsRepository.class.getClassLoader()); cfg.reloadConfiguration();// ww w . ja v a2 s .co m Constructor<?> ctor; Subject subject; try { Class<?> clazz = Class.forName("org.apache.hadoop.security.User"); ctor = clazz.getConstructor(String.class); ctor.setAccessible(true); } catch (ClassNotFoundException | NoSuchMethodException e) { throw new RuntimeException(e); } try { Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name")); subject = new Subject(false, Collections.singleton(principal), Collections.emptySet(), Collections.emptySet()); } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); } // disable file system cache cfg.setBoolean("fs.hdfs.impl.disable.cache", true); // set file system to TestingFs to avoid a bunch of security // checks, similar to what is done in HdfsTests.java cfg.set("fs.AbstractFileSystem." + uri.getScheme() + ".impl", TestingFs.class.getName()); // create the FileContext with our user return Subject.doAs(subject, new PrivilegedAction<FileContext>() { @Override public FileContext run() { try { TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg); return FileContext.getFileContext(fs, cfg); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } } }); }
From source file:org.jwebsocket.plugins.filesystem.FileSystemPlugIn.java
License:Open Source License
private void saveToHDFS(WebSocketConnector aConnector, Token aToken) throws IOException, InterruptedException { if (mLog.isDebugEnabled()) { mLog.debug("Processing 'save to HDFS'..."); }//from w w w . j a va2 s . co m Configuration conf = new Configuration(); conf.setClassLoader(JWebSocketXmlConfigInitializer.getClassLoader()); conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); FileSystem fs = FileSystem.get(URI.create("hdfs://211.189.127.45:8020"), conf, "yarn"); mLog.debug("setup HDFS complete."); TokenServer lServer = getServer(); String lMsg; // check if user is allowed to run 'save' command if (!SecurityFactory.hasRight(lServer.getUsername(aConnector), NS_FILESYSTEM + ".save")) { if (mLog.isDebugEnabled()) { mLog.debug("Returning 'Access denied'..."); } lServer.sendToken(aConnector, lServer.createAccessDenied(aToken)); return; } // instantiate response token Token lResponse = lServer.createResponse(aToken); // obtain required parameters for file load operation String lFilename = aToken.getString("filename"); String lScope = aToken.getString("scope", JWebSocketCommonConstants.SCOPE_PRIVATE); // scope may be "private" or "public" String lBaseDir; if (JWebSocketCommonConstants.SCOPE_PRIVATE.equals(lScope)) { String lUsername = getUsername(aConnector); lBaseDir = getString(PRIVATE_DIR_KEY, PRIVATE_DIR_DEF); if (lUsername != null) { lBaseDir = JWebSocketConfig.expandEnvAndJWebSocketVars(lBaseDir).replace("{username}", lUsername); } else { lMsg = "not authenticated to save private file"; if (mLog.isDebugEnabled()) { mLog.debug(lMsg); } lResponse.setInteger("code", -1); lResponse.setString("msg", lMsg); // send error response to requester lServer.sendToken(aConnector, lResponse); return; } } else if (JWebSocketCommonConstants.SCOPE_PUBLIC.equals(lScope)) { lBaseDir = JWebSocketConfig.expandEnvAndJWebSocketVars(getString(PUBLIC_DIR_KEY, PUBLIC_DIR_DEF)); } else { lMsg = "invalid scope"; if (mLog.isDebugEnabled()) { mLog.debug(lMsg); } lResponse.setInteger("code", -1); lResponse.setString("msg", lMsg); // send error response to requester lServer.sendToken(aConnector, lResponse); return; } Boolean lNotify = aToken.getBoolean("notify", false); String lData = aToken.getString("data"); String lEncoding = aToken.getString("encoding", "base64"); byte[] lBA = null; try { if ("base64".equals(lEncoding)) { int lIdx = lData.indexOf(','); if (lIdx >= 0) { lData = lData.substring(lIdx + 1); } lBA = Base64.decodeBase64(lData); } else { lBA = lData.getBytes("UTF-8"); } } catch (Exception lEx) { mLog.error(Logging.getSimpleExceptionMessage(lEx, "saving file")); } // complete the response token String lFullPath = lBaseDir + lFilename; File lFile = new File(lFullPath); try { // prevent two threads at a time writing to the same file synchronized (this) { // force create folder if not yet exists File lDir = new File(FilenameUtils.getFullPath(lFullPath)); FileUtils.forceMkdir(lDir); if (lBA != null) { FileUtils.writeByteArrayToFile(lFile, lBA); OutputSupplier<? extends OutputStream> os = (OutputSupplier<? extends OutputStream>) fs .create(new Path(fs.getHomeDirectory(), lFilename), true); ByteStreams.write(lBA, os); } else { FileUtils.writeStringToFile(lFile, lData, "UTF-8"); } } } catch (Exception lEx) { lResponse.setInteger("code", -1); lMsg = lEx.getClass().getSimpleName() + " on save: " + lEx.getMessage(); lResponse.setString("msg", lMsg); mLog.error(lMsg); } // send response to requester lServer.sendToken(aConnector, lResponse); // send notification event to other affected clients // to allow to update their content (if desired) if (lNotify) { // create token of type "event" Token lEvent = TokenFactory.createToken(BaseToken.TT_EVENT); // include name space of this plug-in lEvent.setNS(NS_FILESYSTEM); lEvent.setString("name", "filesaved"); lEvent.setString("filename", lFilename); lEvent.setString("sourceId", aConnector.getId()); lEvent.setString("url", getString(WEB_ROOT_KEY, WEB_ROOT_DEF) + lFilename); // TODO: Limit notification to desired scope lServer.broadcastToken(lEvent); } }
From source file:org.springframework.data.hadoop.mapreduce.HadoopCodeExecutor.java
License:Apache License
@SuppressWarnings("unchecked") protected Class<T> resolveTargetClass(Configuration cfg) throws Exception { ClassLoader cl = beanClassLoader; // no target set - we might need to load one from the custom jar if (target == null) { cl = createClassLoaderForJar(jar, cl, cfg); // make sure to pass this to the Configuration cfg.setClassLoader(cl); if (jar != null) { if (log.isTraceEnabled()) { log.trace("Creating custom classloader " + cl); }/*from w w w.j a va 2 s.c o m*/ // fall-back to main if (!StringUtils.hasText(targetClassName)) { String mainClass = ExecutionUtils.mainClass(jar); Assert.notNull(mainClass, "no target class specified and no Main-Class available"); targetClassName = mainClass; if (log.isDebugEnabled()) { log.debug("Discovered Main-Class [" + mainClass + "]"); } } } else { Assert.hasText(targetClassName, "No target object, class or jar specified - execution aborted"); } return loadClass(targetClassName, cl); } return (Class<T>) target.getClass(); }