Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.openflamingo.collector.handler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.openflamingo.collector.JobContext; import org.openflamingo.collector.policy.SelectorPattern; import org.openflamingo.collector.policy.SelectorPatternFactory; import org.openflamingo.model.collector.*; import org.openflamingo.util.DateUtils; import org.openflamingo.util.FileSystemScheme; import org.openflamingo.util.JVMIDUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import static org.openflamingo.util.FileSystemUtils.*; /** * Local FileSystem? ? Ingress . * * @author Edward KIM * @since 0.1 */ public class LocalToHdfsHandler extends DefaultHandler { /** * SLF4J Logging */ private Logger logger = LoggerFactory.getLogger(LocalToHdfsHandler.class); /** * ? ?? ?. ? ? ? ?? ? ?? * ? ?? . */ public static final String PROCESSING_FILE_QUALIFIER = ".processing"; // FIXME /** * Flamingo Log Collector Job Context */ private JobContext jobContext; /** * HDFS File Uploder Job */ private Job job; /** * Job? ?? Ingress */ private FromLocal local; /** * ??. * * @param jobContext Flamingo Log Collector? Job Context * @param job Job */ public LocalToHdfsHandler(JobContext jobContext, Job job) { this.jobContext = jobContext; this.job = job; this.local = job.getPolicy().getIngress().getFromLocal(); } @Override public void execute() throws Exception { // ? ?? ? ??. copyToWorkingDirectory(); // ?? ? ? ?? ??. List<FileStatus> files = getFilesFromWorkingDirectory(); if (files.size() < 1) { logger.info( " ?? ? ? ."); return; } // ? ?? HDFS . Iterator<FileStatus> iterator = files.iterator(); while (iterator.hasNext()) { // ? ?? ? ? ??. FileStatus workingFile = iterator.next(); FileSystem workingFS = getFileSystem(workingFile.getPath()); // ? ?? . String processingFileName = workingFile.getPath().getName() + PROCESSING_FILE_QUALIFIER; String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); Path processingFile = new Path(workingDirectory, processingFileName); boolean renamed = workingFS.rename(workingFile.getPath(), processingFile); logger.debug( " ? ? '{}'? '{}' ?? .", workingFile.getPath(), processingFile); if (renamed) { // Target HDFS ??. ToHdfs hdfs = job.getPolicy().getEgress().getToHdfs(); // ?? HDFS? FileSystem ??. String cluster = jobContext.getValue(hdfs.getCluster()); Configuration configuration = getConfiguration(jobContext.getModel(), cluster); FileSystem targetFS = FileSystem.get(configuration); logger.info( "HDFS? Hadoop Cluster '{}'? Hadoop Cluster? ? ? .", cluster); // HDFS? target, staging . String targetDirectory = jobContext.getValue(hdfs.getTargetPath()); String stagingDirectory = jobContext.getValue(hdfs.getStagingPath()); logger.info( "HDFS? ? '{}'? ? '{}'.", targetDirectory, stagingDirectory); // ? ? ?? . int hash = Math.abs((workingFile.getPath().toString() + processingFile.toString()).hashCode()) + Integer.parseInt(JVMIDUtils.generateUUID()); if (hash < 0) hash = -hash; logger.debug( "? '{}'? ? '{}'? '{}'? ?.", new Object[] { stagingDirectory, processingFile.getName(), hash }); // ? ? . // FIXME Path stagingFile = new Path(stagingDirectory, DateUtils.parseDate(jobContext.getStartDate(), "yyyyMMddHHmmss") + "_" + String.valueOf(hash)); try { targetFS.copyFromLocalFile(false, false, processingFile, stagingFile); } catch (Exception ex) { logger.warn( " ? ? '{}'? ? ? '{}' ? ??.", processingFile, stagingFile); copyToErrorDirectory(workingFile); continue; } logger.info( " ? ? '{}'? ? ? '{}' .", processingFile, stagingFile); // ? ?? ? ??. Path targetFile = new Path(targetDirectory, workingFile.getPath().getName()); targetFS.rename(stagingFile, targetFile); logger.info("? ? '{}' ?? '{}' ??.", stagingFile, targetFile); // ?? ??. copyToCompleteDirectory(workingFS.getFileStatus(processingFile)); } } } @Override public void validate() { ///////////////////////////////// // Ingress :: Local FileSystem ///////////////////////////////// String sourceDirectory = correctPath(jobContext.getValue(local.getSourceDirectory().getPath().trim())); String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); String errorDirectory = correctPath(jobContext.getValue(local.getErrorDirectory().trim())); String completeDirectory = correctPath(jobContext.getValue(local.getCompleteDirectory().trim())); // Scheme? ?. checkScheme(sourceDirectory, FileSystemScheme.LOCAL); checkScheme(workingDirectory, FileSystemScheme.LOCAL); checkScheme(errorDirectory, FileSystemScheme.LOCAL); checkScheme(completeDirectory, FileSystemScheme.LOCAL); // ?? ? ? ?. validateSameFileSystem(sourceDirectory, workingDirectory); validateSameFileSystem(sourceDirectory, errorDirectory); if (completeDirectory != null) { validateSameFileSystem(sourceDirectory, completeDirectory); } // ?. testCreateDir(new Path(sourceDirectory)); testCreateDir(new Path(workingDirectory)); testCreateDir(new Path(errorDirectory)); if (local.getCompleteDirectory() != null) { testCreateDir(new Path(completeDirectory)); } ///////////////////////////////// // Target HDFS ///////////////////////////////// ToHdfs hdfs = job.getPolicy().getEgress().getToHdfs(); String cluster = hdfs.getCluster(); Configuration configuration = this.getConfiguration(jobContext.getModel(), cluster); String stagingPath = jobContext .getValue(configuration.get(HDFS_URL) + "/" + jobContext.getValue(hdfs.getStagingPath())); String targetPath = jobContext .getValue(configuration.get(HDFS_URL) + "/" + jobContext.getValue(hdfs.getTargetPath())); checkScheme(stagingPath, FileSystemScheme.HDFS); checkScheme(targetPath, FileSystemScheme.HDFS); validateSameFileSystem(targetPath, stagingPath); testCreateDir(new Path(stagingPath)); testCreateDir(new Path(targetPath)); } /** * Hadoop Cluster? ? Cluster? Hadoop Configuration? ?. * * @param model Flamingo Log Collector? JAXB ROOT Object * @param clusterName Hadoop Cluster * @return {@link org.apache.hadoop.conf.Configuration} */ public static org.apache.hadoop.conf.Configuration getConfiguration(Collector model, String clusterName) { org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration(); List<Cluster> clusters = model.getClusters().getCluster(); for (Cluster cluster : clusters) { if (clusterName.equals(cluster.getName())) { configuration.set(HDFS_URL, cluster.getFsDefaultName()); configuration.set(JOB_TRACKER, cluster.getMapredJobTracker()); List<Property> properties = cluster.getProperties().getProperty(); for (Property property : properties) { configuration.set(property.getName(), property.getValue()); } } } return configuration; } /** * ?? ?? ? ?? . * * @return ? ?(?? null) * @throws IOException ? ? , ?? ?? */ public List<FileStatus> copyToWorkingDirectory() throws IOException { // ?? ? ? ? Selector Pattern? . SelectorPattern selectorPattern = SelectorPatternFactory.getSelectorPattern( this.local.getSourceDirectory().getConditionType().trim(), jobContext.getValue(this.local.getSourceDirectory().getCondition().trim()), jobContext); String sourceDirectory = correctPath(jobContext.getValue(local.getSourceDirectory().getPath().trim())); String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); FileSystem sourceDirectoryFS = getFileSystem(sourceDirectory); List<FileStatus> files = new LinkedList<FileStatus>(); for (FileStatus sourceFile : sourceDirectoryFS.listStatus(new Path(sourceDirectory))) { if (!sourceFile.isDir()) { if (sourceFile.getPath().getName().startsWith(".") || sourceFile.getPath().getName().startsWith("_") || sourceFile.getPath().getName().endsWith(".work")) { logger.info(" ? '{}'? .", sourceFile.getPath()); continue; } if (selectorPattern.accept(sourceFile.getPath())) { // ?? ??. Path workPath = new Path(workingDirectory, sourceFile.getPath().getName()); sourceDirectoryFS.rename(sourceFile.getPath(), workPath); logger.info("? ? '{}'? '{}' ??.", sourceFile.getPath(), workPath); files.add(sourceDirectoryFS.getFileStatus(workPath)); } } } return files; } /** * ? ?? ?? . ? ? ?? . * ? ?? ? {@link org.openflamingo.collector.handler.LocalToHdfsHandler#PROCESSING_FILE_QUALIFIER} ?. * * @return ? ? ? ? ? ?? ?? ? * @throws IOException ? ? ? ? ?? ? */ public List<FileStatus> getFilesFromWorkingDirectory() throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); List<FileStatus> files = new LinkedList<FileStatus>(); for (FileStatus fs : workingDirectoryFS.listStatus(new Path(workingDirectory))) { if (!fs.isDir()) { if (fs.getPath().getName().endsWith(PROCESSING_FILE_QUALIFIER)) { logger.info("'{}' ?? ? ?? ", fs.getPath()); continue; } files.add(fs); } } return files; } /** * ? ?? ?? . ? ? ? ?? . * ? ?? ? <tt>.processing</tt> ?. * * @return ? ? ? ? ? ?? ? * @throws IOException ? ? ? ? ?? ? */ public List<FileStatus> getProcessingFilesFromWorkingDirectory() throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); List<FileStatus> files = new LinkedList<FileStatus>(); for (FileStatus fs : workingDirectoryFS.listStatus(new Path(workingDirectory))) { if (!fs.isDir()) { if (fs.getPath().getName().endsWith(PROCESSING_FILE_QUALIFIER)) { files.add(fs); } } } return files; } /** * ?? ??. * * @param fileToMove ? * @return ?? ? <tt>true</tt> * @throws IOException ?? ?? */ public boolean copyToCompleteDirectory(FileStatus fileToMove) throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); String completeDirectory = correctPath(jobContext.getValue(local.getCompleteDirectory().trim())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); boolean success = false; if (local.isRemoveAfterCopy()) { logger.info("? . ? ? '{}'? ." + fileToMove.getPath()); success = workingDirectoryFS.delete(fileToMove.getPath(), false); if (!success) { logger.info("? ? '{}'? .", fileToMove.getPath()); } } else { Path completedPath = new Path(completeDirectory, fileToMove.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, "")); logger.info( "? . ? ? '{}'? '{}' ??.", fileToMove.getPath(), completedPath); success = workingDirectoryFS.rename(fileToMove.getPath(), completedPath); if (!success) { logger.warn("? ??? ? ."); } } return success; } /** * ?? ? ??. * * @param fs ? * @return ?? ? <tt>true</tt> * @throws IOException ?? ?? */ public boolean copyToErrorDirectory(FileStatus fs) throws IOException { String workingDirectory = correctPath(jobContext.getValue(local.getWorkingDirectory().trim())); String errorDirectory = correctPath(jobContext.getValue(local.getErrorDirectory().trim())); FileSystem workingDirectoryFS = getFileSystem(workingDirectory); if (fs.getPath().getName().endsWith(PROCESSING_FILE_QUALIFIER)) { Path errorPath = new Path(errorDirectory, fs.getPath().getName().replaceAll(PROCESSING_FILE_QUALIFIER, "")); logger.info( " ? ?? . '{}' ?? '{}' ??.", fs.getPath(), errorPath); return workingDirectoryFS.rename(fs.getPath(), errorPath); } return false; } }