Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package org.apache.streams.hdfs; import org.apache.streams.config.ComponentConfigurator; import org.apache.streams.config.StreamsConfiguration; import org.apache.streams.config.StreamsConfigurator; import org.apache.streams.converter.LineReadWriteUtil; import org.apache.streams.core.DatumStatusCountable; import org.apache.streams.core.DatumStatusCounter; import org.apache.streams.core.StreamsDatum; import org.apache.streams.core.StreamsPersistReader; import org.apache.streams.core.StreamsResultSet; import org.apache.streams.jackson.StreamsJacksonMapper; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Queues; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.security.UserGroupInformation; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.math.BigInteger; import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; /** * WebHdfsPersistReader reads from hdfs. */ public class WebHdfsPersistReader implements StreamsPersistReader, DatumStatusCountable { public static final String STREAMS_ID = "WebHdfsPersistReader"; private static final Logger LOGGER = LoggerFactory.getLogger(WebHdfsPersistReader.class); protected static final char DELIMITER = '\t'; protected FileSystem client; protected Path path; protected FileStatus[] status; protected volatile Queue<StreamsDatum> persistQueue; protected ObjectMapper mapper; protected LineReadWriteUtil lineReaderUtil; protected HdfsReaderConfiguration hdfsConfiguration; protected StreamsConfiguration streamsConfiguration; private ExecutorService executor; protected DatumStatusCounter countersTotal = new DatumStatusCounter(); protected DatumStatusCounter countersCurrent = new DatumStatusCounter(); private Future<?> task; /** * WebHdfsPersistReader constructor - resolves HdfsReaderConfiguration from JVM 'hdfs'. */ public WebHdfsPersistReader() { this(new ComponentConfigurator<>(HdfsReaderConfiguration.class) .detectConfiguration(StreamsConfigurator.getConfig().getConfig("hdfs"))); } /** * WebHdfsPersistReader constructor - uses supplied HdfsReaderConfiguration. * @param hdfsConfiguration hdfsConfiguration */ public WebHdfsPersistReader(HdfsReaderConfiguration hdfsConfiguration) { this.hdfsConfiguration = hdfsConfiguration; } /** * getURI from hdfsConfiguration. * @return URI * @throws URISyntaxException URISyntaxException */ public URI getURI() throws URISyntaxException { StringBuilder uriBuilder = new StringBuilder(); uriBuilder.append(hdfsConfiguration.getScheme()); uriBuilder.append("://"); if (StringUtils.isNotBlank(hdfsConfiguration.getHost())) { uriBuilder.append(hdfsConfiguration.getHost()); if (hdfsConfiguration.getPort() != null) { uriBuilder.append(":" + hdfsConfiguration.getPort()); } } else { uriBuilder.append("/"); } return new URI(uriBuilder.toString()); } /** * isConnected. * @return true if connected, false otherwise */ public boolean isConnected() { return (client != null); } /** * getFileSystem. * @return FileSystem */ public final synchronized FileSystem getFileSystem() { // Check to see if we are connected. if (!isConnected()) { connectToWebHDFS(); } return this.client; } // TODO: combine with WebHdfsPersistReader.connectToWebHDFS private synchronized void connectToWebHDFS() { try { LOGGER.info("User : {}", this.hdfsConfiguration.getUser()); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(this.hdfsConfiguration.getUser()); ugi.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.SIMPLE); ugi.doAs((PrivilegedExceptionAction<Void>) () -> { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); conf.set("fs.file.impl", LocalFileSystem.class.getName()); LOGGER.info("WebURI : {}", getURI().toString()); client = FileSystem.get(getURI(), conf); LOGGER.info("Connected to WebHDFS"); /* * ************************************************************************************************ * This code is an example of how you would work with HDFS and you weren't going over * the webHDFS protocol. * * Smashew: 2013-10-01 * ************************************************************************************************ conf.set("fs.defaultFS", "hdfs://hadoop.mdigitallife.com:8020/user/" + userName); conf.set("namenode.host","0.0.0.0"); conf.set("hadoop.job.ugi", userName); conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, "runner"); fileSystem.createNewFile(new Path("/user/"+ userName + "/test")); FileStatus[] status = fs.listStatus(new Path("/user/" + userName)); for(int i=0;i<status.length;i++) { LOGGER.info("Directory: {}", status[i].getPath()); } */ return null; }); } catch (Exception ex) { LOGGER.error("There was an error connecting to WebHDFS, please check your settings and try again"); ex.printStackTrace(); } } @Override public String getId() { return STREAMS_ID; } @Override public void prepare(Object configurationObject) { LOGGER.debug("Prepare"); lineReaderUtil = LineReadWriteUtil.getInstance(hdfsConfiguration); connectToWebHDFS(); String pathString = hdfsConfiguration.getPath() + "/" + hdfsConfiguration.getReaderPath(); LOGGER.info("Path : {}", pathString); path = new Path(pathString); try { if (client.isFile(path)) { LOGGER.info("Found File"); FileStatus fileStatus = client.getFileStatus(path); status = new FileStatus[1]; status[0] = fileStatus; } else if (client.isDirectory(path)) { status = client.listStatus(path); List<FileStatus> statusList = Arrays.asList(status); Collections.sort(statusList); status = statusList.toArray(new FileStatus[0]); LOGGER.info("Found Directory : {} files", status.length); } else { LOGGER.error("Neither file nor directory, wtf"); } } catch (IOException ex) { LOGGER.error("IOException", ex); } streamsConfiguration = StreamsConfigurator.detectConfiguration(); persistQueue = Queues.synchronizedQueue( new LinkedBlockingQueue<StreamsDatum>(streamsConfiguration.getBatchSize().intValue())); executor = Executors.newSingleThreadExecutor(); mapper = StreamsJacksonMapper.getInstance(); } @Override public void cleanUp() { } @Override public StreamsResultSet readAll() { WebHdfsPersistReaderTask readerTask = new WebHdfsPersistReaderTask(this); Thread readerThread = new Thread(readerTask); readerThread.start(); try { readerThread.join(); } catch (InterruptedException ignored) { LOGGER.trace("ignored InterruptedException", ignored); } return new StreamsResultSet(persistQueue); } @Override public void startStream() { LOGGER.debug("startStream"); task = executor.submit(new WebHdfsPersistReaderTask(this)); } @Override public StreamsResultSet readCurrent() { StreamsResultSet current; synchronized (WebHdfsPersistReader.class) { current = new StreamsResultSet(new ConcurrentLinkedQueue<>(persistQueue)); current.setCounter(new DatumStatusCounter()); current.getCounter().add(countersCurrent); countersTotal.add(countersCurrent); countersCurrent = new DatumStatusCounter(); persistQueue.clear(); } return current; } protected void write(StreamsDatum entry) { boolean success; do { synchronized (WebHdfsPersistReader.class) { success = persistQueue.offer(entry); } Thread.yield(); } while (!success); } @Override public StreamsResultSet readNew(BigInteger sequence) { return null; } @Override public StreamsResultSet readRange(DateTime start, DateTime end) { return null; } @Override public boolean isRunning() { return task == null || !task.isDone() && !task.isCancelled(); } @Override public DatumStatusCounter getDatumStatusCounter() { return countersTotal; } }