Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.nutch.protocol.ftp; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.InetAddress; import java.net.Socket; import java.util.List; import org.apache.commons.net.MalformedServerReplyException; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPCommand; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPFileEntryParser; import org.apache.commons.net.ftp.FTPReply; import org.apache.commons.net.ftp.FTPConnectionClosedException; /*********************************************** * Client.java encapsulates functionalities necessary for nutch to get dir list * and retrieve file from an FTP server. This class takes care of all low level * details of interacting with an FTP server and provides a convenient higher * level interface. * * Modified from FtpClient.java in apache commons-net. * * Notes by John Xing: ftp server implementations are hardly uniform and none * seems to follow RFCs whole-heartedly. We have no choice, but assume common * denominator as following: (1) Use stream mode for data transfer. Block mode * will be better for multiple file downloading and partial file downloading. * However not every ftpd has block mode support. (2) Use passive mode for data * connection. So Nutch will work if we run behind firewall. (3) Data connection * is opened/closed per ftp command for the reasons listed in (1). There are ftp * servers out there, when partial downloading is enforced by closing data * channel socket on our client side, the server side immediately closes control * channel (socket). Our codes deal with such a bad behavior. (4) LIST is used * to obtain remote file attributes if possible. MDTM and SIZE would be nice, but * not as ubiquitously implemented as LIST. (5) Avoid using ABOR in single * thread? Do not use it at all. * * About exceptions: Some specific exceptions are re-thrown as one of * FtpException*.java In fact, each function throws FtpException*.java or pass * IOException. * * @author John Xing ***********************************************/ public class Client extends FTP { private int __dataTimeout; private int __passivePort; private String __passiveHost; // private int __fileType, __fileFormat; private boolean __remoteVerificationEnabled; // private FTPFileEntryParser __entryParser; private String __systemName; /** Public default constructor */ public Client() { __initDefaults(); __dataTimeout = -1; __remoteVerificationEnabled = true; } // defaults when initialize private void __initDefaults() { __passiveHost = null; __passivePort = -1; __systemName = null; // __fileType = FTP.ASCII_FILE_TYPE; // __fileFormat = FTP.NON_PRINT_TEXT_FORMAT; // __entryParser = null; } // parse reply for pass() private void __parsePassiveModeReply(String reply) throws MalformedServerReplyException { int i, index, lastIndex; String octet1, octet2; StringBuffer host; reply = reply.substring(reply.indexOf('(') + 1, reply.indexOf(')')).trim(); host = new StringBuffer(24); lastIndex = 0; index = reply.indexOf(','); host.append(reply.substring(lastIndex, index)); for (i = 0; i < 3; i++) { host.append('.'); lastIndex = index + 1; index = reply.indexOf(',', lastIndex); host.append(reply.substring(lastIndex, index)); } lastIndex = index + 1; index = reply.indexOf(',', lastIndex); octet1 = reply.substring(lastIndex, index); octet2 = reply.substring(index + 1); // index and lastIndex now used as temporaries try { index = Integer.parseInt(octet1); lastIndex = Integer.parseInt(octet2); } catch (NumberFormatException e) { throw new MalformedServerReplyException( "Could not parse passive host information.\nServer Reply: " + reply); } index <<= 8; index |= lastIndex; __passiveHost = host.toString(); __passivePort = index; } /** * open a passive data connection socket * * @param command * @param arg * @return * @throws IOException * @throws FtpExceptionCanNotHaveDataConnection */ protected Socket __openPassiveDataConnection(int command, String arg) throws IOException, FtpExceptionCanNotHaveDataConnection { Socket socket; // // 20040317, xing, accommodate ill-behaved servers, see below // int port_previous = __passivePort; if (pasv() != FTPReply.ENTERING_PASSIVE_MODE) throw new FtpExceptionCanNotHaveDataConnection("pasv() failed. " + getReplyString()); try { __parsePassiveModeReply(getReplyStrings()[0]); } catch (MalformedServerReplyException e) { throw new FtpExceptionCanNotHaveDataConnection(e.getMessage()); } // // 20040317, xing, accommodate ill-behaved servers, see above // int count = 0; // System.err.println("__passivePort "+__passivePort); // System.err.println("port_previous "+port_previous); // while (__passivePort == port_previous) { // // just quit if too many tries. make it an exception here? // if (count++ > 10) // return null; // // slow down further for each new try // Thread.sleep(500*count); // if (pasv() != FTPReply.ENTERING_PASSIVE_MODE) // throw new FtpExceptionCanNotHaveDataConnection( // "pasv() failed. " + getReplyString()); // //return null; // try { // __parsePassiveModeReply(getReplyStrings()[0]); // } catch (MalformedServerReplyException e) { // throw new FtpExceptionCanNotHaveDataConnection(e.getMessage()); // } // } socket = _socketFactory_.createSocket(__passiveHost, __passivePort); if (!FTPReply.isPositivePreliminary(sendCommand(command, arg))) { socket.close(); return null; } if (__remoteVerificationEnabled && !verifyRemote(socket)) { InetAddress host1, host2; host1 = socket.getInetAddress(); host2 = getRemoteAddress(); socket.close(); // our precaution throw new FtpExceptionCanNotHaveDataConnection( "Host attempting data connection " + host1.getHostAddress() + " is not same as server " + host2.getHostAddress() + " So we intentionally close it for security precaution."); } if (__dataTimeout >= 0) socket.setSoTimeout(__dataTimeout); return socket; } /*** * Sets the timeout in milliseconds to use for data connection. set * immediately after opening the data connection. ***/ public void setDataTimeout(int timeout) { __dataTimeout = timeout; } /*** * Closes the connection to the FTP server and restores connection parameters * to the default values. * <p> * * @exception IOException * If an error occurs while disconnecting. ***/ public void disconnect() throws IOException { __initDefaults(); super.disconnect(); // no worry for data connection, since we always close it // in every ftp command that invloves data connection } /*** * Enable or disable verification that the remote host taking part of a data * connection is the same as the host to which the control connection is * attached. The default is for verification to be enabled. You may set this * value at any time, whether the FTPClient is currently connected or not. * <p> * * @param enable * True to enable verification, false to disable verification. ***/ public void setRemoteVerificationEnabled(boolean enable) { __remoteVerificationEnabled = enable; } /*** * Return whether or not verification of the remote host participating in data * connections is enabled. The default behavior is for verification to be * enabled. * <p> * * @return True if verification is enabled, false if not. ***/ public boolean isRemoteVerificationEnabled() { return __remoteVerificationEnabled; } /*** * Login to the FTP server using the provided username and password. * <p> * * @param username * The username to login under. * @param password * The password to use. * @return True if successfully completed, false if not. * @exception FTPConnectionClosedException * If the FTP server prematurely closes the connection as a * result of the client being idle or some other reason causing * the server to send FTP reply code 421. This exception may be * caught either as an IOException or independently as itself. * @exception IOException * If an I/O error occurs while either sending a command to the * server or receiving a reply from the server. ***/ public boolean login(String username, String password) throws IOException { user(username); if (FTPReply.isPositiveCompletion(getReplyCode())) return true; // If we get here, we either have an error code, or an intermmediate // reply requesting password. if (!FTPReply.isPositiveIntermediate(getReplyCode())) return false; return FTPReply.isPositiveCompletion(pass(password)); } /*** * Logout of the FTP server by sending the QUIT command. * <p> * * @return True if successfully completed, false if not. * @exception FTPConnectionClosedException * If the FTP server prematurely closes the connection as a * result of the client being idle or some other reason causing * the server to send FTP reply code 421. This exception may be * caught either as an IOException or independently as itself. * @exception IOException * If an I/O error occurs while either sending a command to the * server or receiving a reply from the server. ***/ public boolean logout() throws IOException { return FTPReply.isPositiveCompletion(quit()); } /** * retrieve list reply for path * * @param path * @param entries * @param limit * @param parser * @throws IOException * @throws FtpExceptionCanNotHaveDataConnection * @throws FtpExceptionUnknownForcedDataClose * @throws FtpExceptionControlClosedByForcedDataClose */ public void retrieveList(String path, List<FTPFile> entries, int limit, FTPFileEntryParser parser) throws IOException, FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose, FtpExceptionControlClosedByForcedDataClose { Socket socket = __openPassiveDataConnection(FTPCommand.LIST, path); if (socket == null) throw new FtpExceptionCanNotHaveDataConnection("LIST " + ((path == null) ? "" : path)); BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); // force-close data channel socket, when download limit is reached // boolean mandatory_close = false; // List entries = new LinkedList(); int count = 0; String line = parser.readNextEntry(reader); while (line != null) { FTPFile ftpFile = parser.parseFTPEntry(line); // skip non-formatted lines if (ftpFile == null) { line = parser.readNextEntry(reader); continue; } entries.add(ftpFile); count += line.length(); // impose download limit if limit >= 0, otherwise no limit // here, cut off is up to the line when total bytes is just over limit if (limit >= 0 && count > limit) { // mandatory_close = true; break; } line = parser.readNextEntry(reader); } // if (mandatory_close) // you always close here, no matter mandatory_close or not. // however different ftp servers respond differently, see below. socket.close(); // scenarios: // (1) mandatory_close is false, download limit not reached // no special care here // (2) mandatory_close is true, download limit is reached // different servers have different reply codes: try { int reply = getReply(); if (!_notBadReply(reply)) throw new FtpExceptionUnknownForcedDataClose(getReplyString()); } catch (FTPConnectionClosedException e) { // some ftp servers will close control channel if data channel socket // is closed by our end before all data has been read out. Check: // tux414.q-tam.hp.com FTP server (hp.com version whp02) // so must catch FTPConnectionClosedException thrown by getReply() above // disconnect(); throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage()); } } /** * retrieve file for path * * @param path * @param os * @param limit * @throws IOException * @throws FtpExceptionCanNotHaveDataConnection * @throws FtpExceptionUnknownForcedDataClose * @throws FtpExceptionControlClosedByForcedDataClose */ public void retrieveFile(String path, OutputStream os, int limit) throws IOException, FtpExceptionCanNotHaveDataConnection, FtpExceptionUnknownForcedDataClose, FtpExceptionControlClosedByForcedDataClose { Socket socket = __openPassiveDataConnection(FTPCommand.RETR, path); if (socket == null) throw new FtpExceptionCanNotHaveDataConnection("RETR " + ((path == null) ? "" : path)); InputStream input = socket.getInputStream(); // 20040318, xing, treat everything as BINARY_FILE_TYPE for now // do we ever need ASCII_FILE_TYPE? // if (__fileType == ASCII_FILE_TYPE) // input = new FromNetASCIIInputStream(input); // fixme, should we instruct server here for binary file type? // force-close data channel socket // boolean mandatory_close = false; int len; int count = 0; byte[] buf = new byte[org.apache.commons.net.io.Util.DEFAULT_COPY_BUFFER_SIZE]; while ((len = input.read(buf, 0, buf.length)) != -1) { count += len; // impose download limit if limit >= 0, otherwise no limit // here, cut off is exactly of limit bytes if (limit >= 0 && count > limit) { os.write(buf, 0, len - (count - limit)); // mandatory_close = true; break; } os.write(buf, 0, len); os.flush(); } // if (mandatory_close) // you always close here, no matter mandatory_close or not. // however different ftp servers respond differently, see below. socket.close(); // scenarios: // (1) mandatory_close is false, download limit not reached // no special care here // (2) mandatory_close is true, download limit is reached // different servers have different reply codes: // do not need this // sendCommand("ABOR"); try { int reply = getReply(); if (!_notBadReply(reply)) throw new FtpExceptionUnknownForcedDataClose(getReplyString()); } catch (FTPConnectionClosedException e) { // some ftp servers will close control channel if data channel socket // is closed by our end before all data has been read out. Check: // tux414.q-tam.hp.com FTP server (hp.com version whp02) // so must catch FTPConnectionClosedException thrown by getReply() above // disconnect(); throw new FtpExceptionControlClosedByForcedDataClose(e.getMessage()); } } /** * reply check after closing data connection * * @param reply * @return */ private boolean _notBadReply(int reply) { if (FTPReply.isPositiveCompletion(reply)) { // do nothing } else if (reply == 426) { // FTPReply.TRANSFER_ABORTED // some ftp servers reply 426, e.g., // foggy FTP server (Version wu-2.6.2(2) // there is second reply witing? no! // getReply(); } else if (reply == 450) { // FTPReply.FILE_ACTION_NOT_TAKEN // some ftp servers reply 450, e.g., // ProFTPD [ftp.kernel.org] // there is second reply witing? no! // getReply(); } else if (reply == 451) { // FTPReply.ACTION_ABORTED // some ftp servers reply 451, e.g., // ProFTPD [ftp.kernel.org] // there is second reply witing? no! // getReply(); } else if (reply == 451) { // FTPReply.ACTION_ABORTED } else { // what other kind of ftp server out there? return false; } return true; } /*** * Sets the file type to be transferred. This should be one of * <code> FTP.ASCII_FILE_TYPE </code>, <code> FTP.IMAGE_FILE_TYPE </code>, * etc. The file type only needs to be set when you want to change the type. * After changing it, the new type stays in effect until you change it again. * The default file type is <code> FTP.ASCII_FILE_TYPE </code> if this method * is never called. * <p> * * @param fileType * The <code> _FILE_TYPE </code> constant indcating the type of file. * @return True if successfully completed, false if not. * @exception FTPConnectionClosedException * If the FTP server prematurely closes the connection as a * result of the client being idle or some other reason causing * the server to send FTP reply code 421. This exception may be * caught either as an IOException or independently as itself. * @exception IOException * If an I/O error occurs while either sending a command to the * server or receiving a reply from the server. ***/ public boolean setFileType(int fileType) throws IOException { if (FTPReply.isPositiveCompletion(type(fileType))) { /* * __fileType = fileType; __fileFormat = FTP.NON_PRINT_TEXT_FORMAT; */ return true; } return false; } /*** * Fetches the system type name from the server and returns the string. This * value is cached for the duration of the connection after the first call to * this method. In other words, only the first time that you invoke this * method will it issue a SYST command to the FTP server. FTPClient will * remember the value and return the cached value until a call to disconnect. * <p> * * @return The system type name obtained from the server. null if the * information could not be obtained. * @exception FTPConnectionClosedException * If the FTP server prematurely closes the connection as a * result of the client being idle or some other reason causing * the server to send FTP reply code 421. This exception may be * caught either as an IOException or independently as itself. * @exception IOException * If an I/O error occurs while either sending a command to the * server or receiving a reply from the server. ***/ public String getSystemName() throws IOException, FtpExceptionBadSystResponse { // if (syst() == FTPReply.NAME_SYSTEM_TYPE) // Technically, we should expect a NAME_SYSTEM_TYPE response, but // in practice FTP servers deviate, so we soften the condition to // a positive completion. if (__systemName == null && FTPReply.isPositiveCompletion(syst())) { __systemName = (getReplyStrings()[0]).substring(4); } else { throw new FtpExceptionBadSystResponse("Bad response of SYST: " + getReplyString()); } return __systemName; } /*** * Sends a NOOP command to the FTP server. This is useful for preventing * server timeouts. * <p> * * @return True if successfully completed, false if not. * @exception FTPConnectionClosedException * If the FTP server prematurely closes the connection as a * result of the client being idle or some other reason causing * the server to send FTP reply code 421. This exception may be * caught either as an IOException or independently as itself. * @exception IOException * If an I/O error occurs while either sending a command to the * server or receiving a reply from the server. ***/ public boolean sendNoOp() throws IOException { return FTPReply.isPositiveCompletion(noop()); } // client.stat(path); // client.sendCommand("STAT"); // client.sendCommand("STAT",path); // client.sendCommand("MDTM",path); // client.sendCommand("SIZE",path); // client.sendCommand("HELP","SITE"); // client.sendCommand("SYST"); // client.setRestartOffset(120); }