Java tutorial
/* -*-Java-*- ******************************************************************************* * * File: NewMain.java * Rev: $Revision: 1.4 $ * Description: webin-data-streamer-Upload-Client * Author: Alexander Senf * Language: Java * * (C) Copyright 2013 EMBL-EBI, all rights reserved. * * This software is released, without warranty, under the terms of * the GNU Public License. See http://www.gnu.org/ * *******************************************************************************/ package uk.ac.ebi.enaega.uploadclient; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import it.sauronsoftware.ftp4j.FTPAbortedException; import it.sauronsoftware.ftp4j.FTPDataTransferException; import it.sauronsoftware.ftp4j.FTPException; import it.sauronsoftware.ftp4j.FTPIllegalReplyException; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.net.InetAddress; import java.net.UnknownHostException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; import java.security.SecureRandom; import java.security.Security; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.TimeZone; import java.util.logging.Level; import java.util.logging.Logger; import javax.swing.JFrame; import net.sf.samtools.BAMIndexer; import net.sf.samtools.SAMFileReader; import net.sf.samtools.SAMRecord; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openpgp.PGPCompressedDataGenerator; import org.bouncycastle.openpgp.PGPEncryptedData; import org.bouncycastle.openpgp.PGPEncryptedDataGenerator; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPLiteralData; import org.bouncycastle.openpgp.PGPLiteralDataGenerator; import org.bouncycastle.openpgp.PGPObjectFactory; import org.bouncycastle.openpgp.PGPPublicKey; import org.bouncycastle.openpgp.PGPPublicKeyRing; import org.bouncycastle.openpgp.PGPPublicKeyRingCollection; import org.bouncycastle.openpgp.PGPUtil; import uk.ac.ebi.ega.reqcheck.ReqCheck; /** * * @author asenf */ public class NewMain { private static final String version = "0.2"; private static String build = "07012014.1"; /** * @param args the command line arguments */ public static void main(String[] args) { System.setProperty("java.net.preferIPv4Stack", "true"); String resource = "/build.version"; try { InputStream in = NewMain.class.getResourceAsStream(resource); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String readLine1 = br.readLine(), readLine2 = br.readLine(), readLine3 = br.readLine(); build = readLine3.substring(readLine3.indexOf("=") + 1) + " (" + readLine2.substring(1) + ")"; } catch (IOException th) { System.out.println(th.getLocalizedMessage()); } if (args.length == 0) { // show login form String[] arg = app(); if (arg != null && arg.length > 0 && arg[0] != null && arg[0].length() > 0) UploadForm.main(arg); else System.exit(0); } else if ((args.length == 1 && args[0].contains("-help")) || (args.length == 2 && args[1].contains("-help"))) { printUsage(); } else if ((args.length == 1 && args[0].contains("-version")) || (args.length == 2 && args[1].contains("-version"))) { System.out.println("Webin Upload Client. Version: " + version + " Build: " + build); } else if (args.length == 3 && (!args[0].contains("-p") && !args[0].contains("-file"))) { // "ena" or "ega" and "ftp.sra.ebi.ac.uk" or "ftp-private.ebi.ac.uk" plus port // Directly show main form String[] arg_ = new String[6]; System.arraycopy(args, 0, arg_, 1, 3); arg_[4] = version; arg_[5] = build; arg_[0] = "true"; UploadForm.main(arg_); } else { // Command-line operation if (args[0].equalsIgnoreCase("-p") || args[0].equalsIgnoreCase("-pp")) { processParmsParser(args); } else if (args[0].equalsIgnoreCase("-file")) { processFileParm(args); } else { String[] args_ = null; if (args != null && args.length > 0 && args[0] != null && args[0].length() > 0) { if (args[0].equalsIgnoreCase("-open")) { args_ = new String[args.length - 1]; System.arraycopy(args, 1, args_, 0, args.length - 1); } else { args_ = new String[args.length]; System.arraycopy(args, 0, args_, 0, args.length); } UploadForm.main(args_); } else { System.exit(0); } } } } // ************************************************************************* // ************************************************************************* public static class JCommander_List { @Parameter List<String> parameters = new ArrayList<String>(); @Parameter(names = { "-p", "-P" }, variableArity = true, description = "Parameter Mode") boolean singlethread; @Parameter(names = { "-pp", "-PP" }, variableArity = true, description = "Parallel Parameter Mode") Integer threads = 1; @Parameter(names = { "-user", "-username" }, description = "Username") String username; @Parameter(names = { "-pass", "-password" }, description = "Password") String password; @Parameter(names = { "-fm", "-filemode" }, description = "Upload-to-file mode") boolean filemode = false; @Parameter(names = { "-v", "-V" }, description = "Verbose mode") boolean v = false; @Parameter(names = { "-vv", "-VV" }, description = "Very Verbose mode") boolean vv = false; @Parameter(names = { "-o", "-O" }, description = "Overwrite files on the server") boolean o = false; @Parameter(names = { "-index", "-Index" }, description = "Create Index file for BAM file") boolean index = false; @Parameter(names = { "-f", "-file", "-files" }, variableArity = true, description = "Filename(s) (to be downloaded/decrypted/indexed)") List<String> files = new ArrayList<String>(); @Parameter(names = "-udt", description = "Enable UDT, if available") boolean udt = false; } private static void processParmsParser(String[] args) { JCommander_List jct = new JCommander_List(); JCommander jCommander = new JCommander(jct, args); System.out.println(); // Numper of parallel processing threads, if PP is selected int numThreads = 1; if (!jct.singlethread) numThreads = jct.threads; boolean verbose = jct.v, very = jct.vv; List<String> files_ = jct.files; int numfiles = files_.size(); String[] files = new String[numfiles]; for (int i = 0; i < numfiles; i++) files[i] = files_.get(i); //HashSet filenames = new HashSet(); //filenames.addAll(Arrays.asList(files)); FileWriter log = null; DateFormat df = new SimpleDateFormat("dd_MMM_yyyy_HH_mm_ss"); df.setTimeZone(TimeZone.getTimeZone("Europe/London")); String now = df.format(System.currentTimeMillis()); String logpath = "upload_log_" + now + ".log"; try { log = new FileWriter(logpath); if (verbose) System.out.println("Creating log file: " + logpath); } catch (IOException ex) { Logger.getLogger(TransferClient.class.getName()).log(Level.SEVERE, null, ex); log = null; if (verbose) System.out.println("Can't create log file: " + ex.getMessage()); } // Command; validate each command individually boolean success = true; if (jct.filemode) { // Upload-to-file boolean index = jct.index; uploadFile(log, verbose, very, files, index); } else { // Upload-to-ebi String username = jct.username; String password = jct.password; String server = "sra-ftps.ebi.ac.uk"; try { server = InetAddress.getByName(server).getHostAddress(); } catch (UnknownHostException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } int port = -1; if (username.toLowerCase().contains("era") || username.toLowerCase().startsWith("webin")) { port = 8021; } else if (username.toLowerCase().contains("crg")) { server = "84.88.52.70"; port = 21; } else if (username.toLowerCase().contains("ega")) { port = 8121; if (!ReqCheck.reqCheck_cmdl(true)) { System.out.println("Your system doesn't meet the minimum upload requirements"); return; } } boolean UDT = jct.udt; boolean overwrite = jct.o; upload(server, port, username, password, log, verbose, very, UDT, numThreads, files, overwrite); } if (log != null) try { log.close(); } catch (IOException ex) { } } private static void upload(String server, int port, String username, String password, FileWriter log, boolean verbose, boolean very, boolean udt, int numThreads, String[] files, boolean over) { try { FTPClient_alternative myClient = new FTPClient_alternative(); String[] connect = myClient.connect(server, port); if (!checkVersion(connect)) return; if (very && log != null) log.write("Connecting to Server " + server + "\n"); if (very && log != null) for (int i = 0; i < connect.length; log.write("Result: " + i + " " + connect[i++])) ; myClient.login(username, password); if (very && log != null) log.write("Logging in as " + username + "\n"); PGPPublicKey pgKey = null; if (username.contains("ega")) { InputStream in = UploadApplet.class.getResourceAsStream("/pubring.gpg"); Security.addProvider(new BouncyCastleProvider()); try { pgKey = readPublicKey(in); if (log != null) log.write("EGA upload. Encryption active.\n"); if (verbose) System.out.println("EGA Public Key read. Upload will be encrypted."); } catch (PGPException ex) { if (verbose && log != null) { log.write("PGPException: " + ex.toString() + "\n"); System.out.println("PGP Exception: " + ex.getMessage()); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } } } } else if (username.contains("crg")) { InputStream in = UploadApplet.class.getResourceAsStream("/exported.gpg"); Security.addProvider(new BouncyCastleProvider()); try { pgKey = getEncryptionKey(getKeyring(in)); if (log != null) log.write("CRG upload. Encryption active.\n"); if (verbose) System.out.println("CRG Public Key read. Upload will be encrypted."); } catch (IOException ex) { if (verbose && log != null) { log.write("Exception: " + ex.toString() + "\n"); System.out.println("Exception: " + ex.getMessage()); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } } } } if (udt) { if (!myClient.isUDTSupported()) { if (log != null) log.write("Server does not support UDT protocol.\n"); if (verbose) System.out.println("Server does not support UDT protocol."); udt = false; } else { myClient.setSockType(true); if (myClient.isUDP()) { if (log != null) log.write("UDT data transfer protocol selected. Limit: 1 thread.\n"); if (verbose) System.out.println("UDT data transfer protocol selected. Limit: 1 thread."); numThreads = 1; } else { udt = false; } } } int uploaded_files = 0, existing_files = 0, error_files = 0; FTPUploadThread my_threads[] = new FTPUploadThread[numThreads]; int count = 0, savecount = 0, size = files.length; boolean alive = true; do { ArrayList indices = new ArrayList(); // indices of "free" threads alive = false; for (int i = 0; i < numThreads; i++) { // find threads that have ended if ((my_threads[i] != null) && (my_threads[i].isAlive())) { alive = true; } else { indices.add(i); if (my_threads[i] != null) { int r = processCompleted(my_threads[i].getStatus(), my_threads[i].getMD5(), my_threads[i].getFilename(), log, myClient, verbose, very, my_threads[i].getInfoPath()); if (r == 1) uploaded_files++; if (r == 2) existing_files++; if (r == 3) error_files++; my_threads[i] = null; savecount++; // count completed threads } } } // Previous loop determined free threads; fill them in the next loop if (indices.size() > 0 && count < size) { // If there are open threads, then for (int i = 0; i < indices.size(); i++) { // Fill all open spaces if (count < size) { // Catch errors int index = Integer.parseInt(indices.get(i).toString()); my_threads[index] = new FTPUploadThread(myClient, pgKey, files[count], index, over); my_threads[index].start(); // start the learning algorithm (thread) count++; // count started threads } } } // runs until the number of completed threads equals the number of files, and all threads completed (redundant) } while ((savecount < size) || alive); myClient.logout(); myClient = null; if (log != null) log.write("\n\nUpload of " + uploaded_files + " file(s) Completed.\n"); if (verbose) System.out.println("Upload of " + uploaded_files + " file(s) Completed."); if (existing_files > 0) { if (log != null) log.write(existing_files + " file(s) already existed in the dropbox, they were skipped.\n"); if (verbose) System.out.println( existing_files + " file(s) already existed in the dropbox, they were skipped."); } if (error_files > 0) { if (log != null) log.write(error_files + " file(s) had MD5 issues and should be uploaded again.\n"); if (verbose) System.out.println(error_files + " file(s) had MD5 issues and should be uploaded again."); } } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IllegalStateException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IOException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPIllegalReplyException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPIllegalReplyException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } } private static void uploadFile(FileWriter log, boolean verbose, boolean very, String[] files, boolean indx) { try { PGPPublicKey pgKey = null; InputStream in = UploadApplet.class.getResourceAsStream("/pubring.gpg"); Security.addProvider(new BouncyCastleProvider()); try { pgKey = readPublicKey(in); log.write("EGA upload. Encryption active.\n"); if (verbose) System.out.println("EGA Public Key read. Upload will be encrypted."); // TODO: CRG } catch (PGPException ex) { if (verbose && log != null) { log.write("PGPException: " + ex.toString() + "\n"); System.out.println("PGP Exception: " + ex.getMessage()); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } } } //pgKey = readPublicKey(in); if (log != null) log.write("EGA File Preparation. Encryption active.\n"); if (verbose) System.out.println("EGA Public Key read. File conversion will be encrypted."); int uploaded_files = 0, existing_files = 0, error_files = 0; for (String upName : files) { FileInputStream in_s = new FileInputStream(new File(upName)); boolean exist = false; // overwrite functionality - check if file already exists in selected dir if (!exist) { String[] MD5 = convert((pgKey != null ? upName + ".gpg" : upName), in_s, 0, 0, 0, pgKey); if (MD5 != null && MD5[0].equalsIgnoreCase("Cipher Issues")) { if (log != null) log.write( "Can't instantiate cipher key. Install JCE Unlimited Strength Security Policy!\n"); if (verbose) System.out.println( "Can't instantiate cipher key. Is JCE Unlimited Strength Security Policy installed?\n"); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } if (log != null) log.close(); System.exit(99); } if (log != null) log.write("Converted file: " + upName + ".\tMD5 Sum: " + MD5[0] + "\n"); if (verbose) System.out.println("Converted file: " + upName); if (MD5 != null && MD5.length > 1) { if (log != null) log.write("\tPGP Encrypted MD5: " + MD5[1] + "\n"); } if (MD5 != null && !MD5[0].contains("Error") && !exist) { // upload complete? in that case there's an MD5 sum // Place MD5 ("same_name.MD5") File md5file = new File(upName + ".md5"); md5file.createNewFile(); FileWriter md5_out = new FileWriter(md5file); md5_out.write(MD5[0]); md5_out.close(); // Place Crypto-MD5 ("same_name.MD5") if (MD5.length > 1) { File md5fileCrypt = new File(upName + ".gpg.md5"); md5fileCrypt.createNewFile(); FileWriter md5_cout = new FileWriter(md5fileCrypt); md5_cout.write(MD5[1]); md5_cout.close(); // New: Create index, unless excluded per parameter call if (upName.toUpperCase().endsWith(".BAM") && indx) { String res = createIndex(upName); if (res != null) { MessageDigest digest = null; // calculate checksum of incoming data stream try { digest = MessageDigest.getInstance("MD5"); // get the hash algorithm } catch (NoSuchAlgorithmException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } File ff = new File(upName + ".bai"); if (digest != null && ff.exists()) { FileInputStream ins = new FileInputStream(ff); byte[] buffer = new byte[64000]; int l = 0; while ((l = ins.read(buffer)) != -1) { digest.update(buffer, 0, l); // Calculate MD5 for TCP stream } byte[] md5sum = digest.digest(); String m5 = ""; for (int i_ = 0; i_ < md5sum.length; i_++) m5 += Integer.toString((md5sum[i_] & 0xff) + 0x100, 16).substring(1); File m5fileIdx = new File(upName + ".bai.md5"); m5fileIdx.createNewFile(); FileWriter m5_out = new FileWriter(m5fileIdx); m5_out.write(m5); m5_out.close(); if (log != null) log.write("\tFile: " + upName + ".bai index file generated. MD5:" + m5 + "\n"); } } else { if (log != null) log.write("\tBAM File: " + upName + " ERROR: index file could not be generated!\n"); } } } uploaded_files++; } else { if (log != null) log.write("File: " + upName + " conversion error. MD5 issues." + "\n"); if (verbose) System.out.println("File: " + upName + " upload error. MD5issues."); error_files++; } if (log != null) log.write("\n"); } else { if (log != null) log.write("File: " + upName + " already converted. Upload skipped, no MD5 Sum calculated" + "\n"); if (verbose) System.out.println( "File: " + upName + " already converted. Upload skipped, no MD5 Sum calculated"); existing_files++; } } if (log != null) log.write("\n\nConversion of " + uploaded_files + " file(s) Completed.\n"); if (verbose) System.out.println("Conversion of " + uploaded_files + " file(s) Completed."); if (existing_files > 0) { log.write(existing_files + " file(s) already existed, they were skipped.\n"); if (verbose) System.out.println(existing_files + " file(s) already existed, they were skipped."); } if (error_files > 0) { log.write(error_files + " file(s) had MD5 issues and should be converted again.\n"); if (verbose) System.out.println(error_files + " file(s) had MD5 issues and should be converted again."); } } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IllegalStateException: " + ex.toString()); } catch (IOException ex1) { } } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IOException: " + ex.toString()); } catch (IOException ex1) { } } } // ************************************************************************* private static void processFileParm(String[] args) { // This code runs the upload via command line, without any graphical components int pre_params = 1; boolean verbose = false, very = false; String verb_ = args[1]; if (verb_.equalsIgnoreCase("-v")) { pre_params++; verbose = true; } else if (verb_.equalsIgnoreCase("-vv")) { // very verbose pre_params++; verbose = true; very = true; } verb_ = args[pre_params]; boolean indx = false; if (verb_.toLowerCase().equals("-Index".toLowerCase())) { indx = true; pre_params++; } int numfiles = args.length - pre_params; String[] files = new String[numfiles]; System.arraycopy(args, pre_params, files, 0, numfiles); FileWriter log = null; DateFormat df = new SimpleDateFormat("dd_MMM_yyyy_HH_mm_ss"); df.setTimeZone(TimeZone.getTimeZone("Europe/London")); String now = df.format(System.currentTimeMillis()); String logpath = "conversion_log_" + now + ".log"; try { log = new FileWriter(logpath); if (verbose) System.out.println("Creating log file: " + logpath); } catch (IOException ex) { Logger.getLogger(TransferClient.class.getName()).log(Level.SEVERE, null, ex); log = null; } try { PGPPublicKey pgKey = null; InputStream in = UploadApplet.class.getResourceAsStream("/pubring.gpg"); Security.addProvider(new BouncyCastleProvider()); try { pgKey = readPublicKey(in); if (log != null) log.write("EGA upload. Encryption active.\n"); if (verbose) System.out.println("EGA Public Key read. Upload will be encrypted."); // TODO: CRG } catch (PGPException ex) { if (verbose && log != null) { log.write("PGPException: " + ex.toString() + "\n"); System.out.println("PGP Exception: " + ex.getMessage()); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } } } //pgKey = readPublicKey(in); if (log != null) log.write("EGA File Preparation. Encryption active.\n"); if (verbose) System.out.println("EGA Public Key read. File conversion will be encrypted."); int uploaded_files = 0, existing_files = 0, error_files = 0; for (String upName : files) { FileInputStream in_s = new FileInputStream(new File(upName)); boolean exist = false; // overwrite functionality - check if file already exists in selected dir if (!exist) { String[] MD5 = convert((pgKey != null ? upName + ".gpg" : upName), in_s, 0, 0, 0, pgKey); if (MD5 != null && MD5[0].equalsIgnoreCase("Cipher Issues")) { if (log != null) log.write( "Can't instantiate cipher key. Install JCE Unlimited Strength Security Policy!\n"); if (verbose) System.out.println( "Can't instantiate cipher key. Is JCE Unlimited Strength Security Policy installed?\n"); if (very && log != null) { Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { log.write(line + "\n"); System.out.print(line); } } if (log != null) log.close(); System.exit(99); } if (log != null) log.write("Converted file: " + upName + ".\tMD5 Sum: " + MD5[0] + "\n"); if (verbose) System.out.println("Converted file: " + upName); if (MD5 != null && MD5.length > 1) { if (log != null) log.write("\tPGP Encrypted MD5: " + MD5[1] + "\n"); } if (MD5 != null && !MD5[0].contains("Error") && !exist) { // upload complete? in that case there's an MD5 sum // Place MD5 ("same_name.MD5") File md5file = new File(upName + ".md5"); md5file.createNewFile(); FileWriter md5_out = new FileWriter(md5file); md5_out.write(MD5[0]); md5_out.close(); // Place Crypto-MD5 ("same_name.MD5") if (MD5.length > 1) { File md5fileCrypt = new File(upName + ".gpg.md5"); md5fileCrypt.createNewFile(); FileWriter md5_cout = new FileWriter(md5fileCrypt); md5_cout.write(MD5[1]); md5_cout.close(); // New: Create index, unless excluded per parameter call if (upName.toUpperCase().endsWith(".BAM") && indx) { String res = createIndex(upName); if (res != null) { MessageDigest digest = null; // calculate checksum of incoming data stream try { digest = MessageDigest.getInstance("MD5"); // get the hash algorithm } catch (NoSuchAlgorithmException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } File ff = new File(upName + ".bai"); if (digest != null && ff.exists()) { FileInputStream ins = new FileInputStream(ff); byte[] buffer = new byte[64000]; int l = 0; while ((l = ins.read(buffer)) != -1) { digest.update(buffer, 0, l); // Calculate MD5 for TCP stream } byte[] md5sum = digest.digest(); String m5 = ""; for (int i_ = 0; i_ < md5sum.length; i_++) m5 += Integer.toString((md5sum[i_] & 0xff) + 0x100, 16).substring(1); File m5fileIdx = new File(upName + ".bai.md5"); m5fileIdx.createNewFile(); FileWriter m5_out = new FileWriter(m5fileIdx); m5_out.write(m5); m5_out.close(); if (log != null) log.write("\tFile: " + upName + ".bai index file generated. MD5:" + m5 + "\n"); } } else { if (log != null) log.write("\tBAM File: " + upName + " ERROR: index file could not be generated!\n"); } } } uploaded_files++; } else { if (log != null) log.write("File: " + upName + " conversion error. MD5 issues." + "\n"); if (verbose) System.out.println("File: " + upName + " upload error. MD5issues."); error_files++; } if (log != null) log.write("\n"); } else { if (log != null) log.write("File: " + upName + " already converted. Upload skipped, no MD5 Sum calculated" + "\n"); if (verbose) System.out.println( "File: " + upName + " already converted. Upload skipped, no MD5 Sum calculated"); existing_files++; } } if (log != null) log.write("\n\nConversion of " + uploaded_files + " file(s) Completed.\n"); if (verbose) System.out.println("Conversion of " + uploaded_files + " file(s) Completed."); if (existing_files > 0) { if (log != null) log.write(existing_files + " file(s) already existed, they were skipped.\n"); if (verbose) System.out.println(existing_files + " file(s) already existed, they were skipped."); } if (error_files > 0) { if (log != null) log.write(error_files + " file(s) had MD5 issues and should be converted again.\n"); if (verbose) System.out.println(error_files + " file(s) had MD5 issues and should be converted again."); } if (log != null) log.close(); } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IllegalStateException: " + ex.toString()); } catch (IOException ex1) { } } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IOException: " + ex.toString()); } catch (IOException ex1) { } } if (log != null) try { log.close(); } catch (IOException ex) { } } private static String[] convert(String fileName, InputStream inputStream, long restartAt, long streamOffset, long offset, PGPPublicKey pgKey) { String result = "Error", result1 = "Error"; String[] results = null; // If public Key provided - use it to encrypt file (provider already added) ByteArrayOutputStream baos = null; // PGP MessageDigest crypt_digest = null; // PGP OutputStream literalOut = null, encOut = null, compressedOut = null; // PGP int DEFAULT_BUFFER_SIZE = 65 * 1024; // PGP PGPEncryptedDataGenerator encryptedDataGenerator = null; // PGP PGPCompressedDataGenerator compressedDataGenerator = null; // PGP PGPLiteralDataGenerator literalDataGenerator = null; // PGP if (pgKey != null) { try { Security.addProvider(new BouncyCastleProvider()); baos = new ByteArrayOutputStream(2 * DEFAULT_BUFFER_SIZE); // Write to memory! try { crypt_digest = MessageDigest.getInstance("MD5"); } catch (NoSuchAlgorithmException ex) { Logger.getLogger(FTPClient_alternative.class.getName()).log(Level.SEVERE, null, ex); System.out.println("MD5 Algorithm not installed with Java. Contact Systems."); System.out.print(ex.getLocalizedMessage()); } // Encrypted Data Generator -- needs unlimited Security Policy encryptedDataGenerator = new PGPEncryptedDataGenerator(PGPEncryptedData.CAST5, true, new SecureRandom(), "BC"); try { encryptedDataGenerator.addMethod(pgKey); encOut = encryptedDataGenerator.open(baos, new byte[DEFAULT_BUFFER_SIZE]); } catch (NoSuchProviderException ex) { Logger.getLogger(FTPClient_alternative.class.getName()).log(Level.SEVERE, null, ex); System.out.println("No Such Service Provider Error: " + ex.getLocalizedMessage()); } catch (PGPException ex) { Logger.getLogger(FTPClient_alternative.class.getName()).log(Level.SEVERE, null, ex); System.out.println("PGP Error: " + ex.getLocalizedMessage()); System.out.println("Ensure that Unlimited Strength Policy files are installed for this JRE:"); Process java = Runtime.getRuntime().exec("cmd /C java -version"); BufferedReader in_ = new BufferedReader(new InputStreamReader(java.getInputStream())); String line; while ((line = in_.readLine()) != null) { System.out.print(line); } } // Compression compressedDataGenerator = new PGPCompressedDataGenerator(PGPCompressedDataGenerator.ZIP); compressedOut = compressedOut = new BufferedOutputStream(compressedDataGenerator.open(encOut)); // Literal Data Generator and Output Stream literalDataGenerator = new PGPLiteralDataGenerator(); literalOut = literalDataGenerator.open(compressedOut, PGPLiteralData.BINARY, fileName, new Date(), new byte[DEFAULT_BUFFER_SIZE]); // 1<<16 } catch (IOException t) { String[] failure = { "Cipher Issues" }; return failure; } } // Convert the stream. if (literalOut == null) return null; if (baos == null) return null; try { // Skips. inputStream.skip(streamOffset); OutputStream dataTransferOutputStream = new BufferedOutputStream(new FileOutputStream(fileName)); // Let's do it! byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; int l = 0; MessageDigest digest = null; // calculate checksum of incoming data stream try { digest = MessageDigest.getInstance("MD5"); // get the hash algorithm } catch (NoSuchAlgorithmException ex) { Logger.getLogger(FTPClient_alternative.class.getName()).log(Level.SEVERE, null, ex); System.out.println("MD5 Algorithm not installed with Java. Contact Systems."); System.out.print(ex.getLocalizedMessage()); } while ((l = inputStream.read(buffer)) != -1) { // The actual data transfer loop ------------------- if (pgKey != null) { literalOut.write(buffer, 0, l); // Write to mem buffer literalOut.flush(); byte[] buffer_ = baos.toByteArray(); // retrieve that buffer if (crypt_digest != null) crypt_digest.update(buffer_); // update crypto MD5 dataTransferOutputStream.write(buffer_); // Write cipher data to file baos.reset(); // empty mem buffer } else { dataTransferOutputStream.write(buffer, 0, l); // no cipher -- write directly to socket } //dataTransferOutputStream.flush(); if (digest != null) digest.update(buffer, 0, l); // Calculate plain text MD5 for TCP stream } // Get a representation of the MD5 digest (checksum) plain file byte[] md5sum = digest == null ? null : digest.digest(); result = ""; if (md5sum != null) for (int i = 0; i < md5sum.length; i++) result += Integer.toString((md5sum[i] & 0xff) + 0x100, 16).substring(1); // Upload loop complete. Close PGP streams, if used if (pgKey != null) { literalOut.close(); if (literalDataGenerator != null) literalDataGenerator.close(); // Close all other streams if (compressedOut != null) compressedOut.close(); if (compressedDataGenerator != null) compressedDataGenerator.close(); if (encOut != null) encOut.close(); if (encryptedDataGenerator != null) encryptedDataGenerator.close(); byte[] buffer_ = baos.toByteArray(); // retrieve that buffer if (crypt_digest != null) crypt_digest.update(buffer_); // update crypto MD5 dataTransferOutputStream.write(buffer_); // Write cipher data to socket dataTransferOutputStream.flush(); baos.close(); // Get a representation of the MD5 digest (checksum) cipher file byte[] md5sum_ = crypt_digest == null ? null : crypt_digest.digest(); result1 = ""; if (md5sum_ != null) for (int i = 0; i < md5sum_.length; i++) result1 += Integer.toString((md5sum_[i] & 0xff) + 0x100, 16).substring(1); } } catch (IOException e) { ; } if (pgKey != null) { results = new String[2]; results[0] = result; results[1] = result1; } else { results = new String[1]; results[0] = result; } return results; } private static synchronized int processCompleted(String stat, String[] MD5, String upName, FileWriter log, FTPClient_alternative myClient, boolean verbose, boolean very, String info) { int result = 0; try { if (stat != null && stat.equalsIgnoreCase("skipped")) { log.write("File: " + upName + " already uploaded. Upload skipped, no MD5 Sum calculated" + "\n"); if (verbose) System.out.println( "File: " + upName + " already uploaded. Upload skipped, no MD5 Sum calculated"); result = 2; // existing } else if (stat != null && stat.contains("Error")) { log.write("File: " + upName + " upload error. MD5issues." + "\n"); if (verbose) System.out.println("File: " + upName + " upload error. MD5issues."); result = 3; // error } else if (stat != null && stat.equalsIgnoreCase("Cipher Issues")) { log.write("Can't instantiate cipher key. Install JCE Unlimited Strength Security Policy!\n"); if (verbose) System.out.println( "Can't instantiate cipher key. Is JCE Unlimited Strength Security Policy installed?\n"); log.close(); System.exit(99); } else if (MD5 != null) { log.write("Uploaded file: " + upName + ".\tMD5 Sum: " + MD5[0] + "\n"); if (verbose) System.out.println("Uploaded file: " + upName); if (MD5.length > 1) { log.write("\tPGP Encrypted MD5: " + MD5[1] + "\n"); } result = 1; // uploaded } if (MD5 != null && !MD5[0].contains("Error")) { // upload complete? in that case there's an MD5 sum // Place MD5 on server ("same_name.MD5") File md5file = new File(upName + ".md5"); md5file.createNewFile(); FileWriter md5_out = new FileWriter(md5file); md5_out.write(MD5[0]); md5_out.close(); myClient.upload(md5file); md5file.delete(); // Place Crypto-MD5 on server ("same_name.MD5") if (MD5.length > 1) { File md5fileCrypt = new File(upName + ".gpg.md5"); md5fileCrypt.createNewFile(); FileWriter md5_cout = new FileWriter(md5fileCrypt); md5_cout.write(MD5[1]); md5_cout.close(); myClient.upload(md5fileCrypt); md5fileCrypt.delete(); // New: Create index, upload if (upName.toUpperCase().endsWith(".BAM")) { String m5 = createAndUploadIndex(upName, myClient); if (m5 != null) { File m5fileIdx = new File(upName + ".bai.md5"); m5fileIdx.createNewFile(); FileWriter m5_out = new FileWriter(m5fileIdx); m5_out.write(m5); m5_out.close(); myClient.upload(m5fileIdx); m5fileIdx.delete(); log.write("\tFile: " + upName + ".bai index file generated. MD5:" + m5 + "\n"); } else { log.write("\tBAM File: " + upName + " ERROR: index file could not be generated!\n"); } } } if (info != null) { // VCF Header Info File - upload, upload MD5 String infmd5 = null; try { FileInputStream ins = new FileInputStream(new File(info)); String[] r = myClient.upload(info, ins, 0, 0, null, 0, null, null); infmd5 = r[0]; } catch (IllegalStateException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPIllegalReplyException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPDataTransferException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPAbortedException ex) { Logger.getLogger(UploaderAPI.class.getName()).log(Level.SEVERE, null, ex); } if (infmd5 != null) { File m5fileIdx = new File(info + ".md5"); m5fileIdx.createNewFile(); FileWriter m5_out = new FileWriter(m5fileIdx); m5_out.write(infmd5); m5_out.close(); myClient.upload(m5fileIdx); m5fileIdx.delete(); if (log != null) log.write("\tFile: " + info + " index file generated. MD5:" + infmd5 + "\n"); } else { if (log != null) log.write("\tFile: " + info + " ERROR: index file could not be generated!\n"); } } } log.write("\n"); } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IllegalStateException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FileNotFoundException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FileNotFoundException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPIllegalReplyException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPIllegalReplyException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPDataTransferException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPDataTransferException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (FTPAbortedException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("FTPAbortedException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); if (very && log != null) try { log.write("IOException: " + ex.toString() + "\n"); } catch (IOException ex1) { } } return result; } private static void test1() { try { FTPClient_alternative myClient = new FTPClient_alternative(); myClient.connect("ftp-private.ebi.ac.uk"); myClient.login("---", "---"); InputStream in = UploadApplet.class.getResourceAsStream("/EGA_Public_key.dat"); Security.addProvider(new BouncyCastleProvider()); PGPPublicKey pgKey = readPublicKey(in); String upName = "/home/asenf/README.txt"; File up = new File(upName); FileInputStream in_s = new FileInputStream(up); String[] MD5 = myClient.upload((up.getName() + ".gpg"), in_s, 0, 0, null, 0, null, pgKey); // Place MD5 on server ("same_name.MD5") File md5file = new File(upName + ".md5"); md5file.createNewFile(); FileWriter md5_out = new FileWriter(md5file); md5_out.write(MD5[0]); md5_out.close(); myClient.upload(md5file); md5file.delete(); // Place Crypto-MD5 on server ("same_name.MD5") if (MD5.length > 1) { File md5fileCrypt = new File(upName + ".gpg.md5"); md5fileCrypt.createNewFile(); FileWriter md5_cout = new FileWriter(md5fileCrypt); md5_cout.write(MD5[1]); md5_cout.close(); myClient.upload(md5fileCrypt); md5fileCrypt.delete(); } } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPIllegalReplyException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (PGPException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPDataTransferException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPAbortedException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } } private static PGPPublicKey readPublicKey(InputStream in) throws IOException, PGPException { in = PGPUtil.getDecoderStream(in); PGPPublicKeyRingCollection pgpPub = new PGPPublicKeyRingCollection(in); // // we just loop through the collection till we find a key suitable for encryption, in the real // world you would probably want to be a bit smarter about this. // PGPPublicKey key = null; // // iterate through the key rings. // Iterator rIt = pgpPub.getKeyRings(); while (key == null && rIt.hasNext()) { PGPPublicKeyRing kRing = (PGPPublicKeyRing) rIt.next(); Iterator kIt = kRing.getPublicKeys(); boolean encryptionKeyFound = false; while (key == null && kIt.hasNext()) { PGPPublicKey k = (PGPPublicKey) kIt.next(); if (k.isEncryptionKey()) { key = k; } } } if (key == null) { throw new IllegalArgumentException("Can't find encryption key in key ring."); } return key; } private static PGPPublicKeyRing getKeyring(InputStream keyBlockStream) throws IOException { // PGPUtil.getDecoderStream() will detect ASCII-armor automatically and decode it, // the PGPObject factory then knows how to read all the data in the encoded stream PGPObjectFactory factory = new PGPObjectFactory(PGPUtil.getDecoderStream(keyBlockStream)); // these files should really just have one object in them, // and that object should be a PGPPublicKeyRing. Object o = factory.nextObject(); if (o instanceof PGPPublicKeyRing) { return (PGPPublicKeyRing) o; } throw new IllegalArgumentException("Input text does not contain a PGP Public Key"); } /** * Get the first encyption key off the given keyring. */ private static PGPPublicKey getEncryptionKey(PGPPublicKeyRing keyRing) { if (keyRing == null) return null; // iterate over the keys on the ring, look for one // which is suitable for encryption. Iterator keys = keyRing.getPublicKeys(); PGPPublicKey key = null; while (keys.hasNext()) { key = (PGPPublicKey) keys.next(); if (key.isEncryptionKey()) { return key; } } return null; } // This is the location where the server is specified in command line mode private static String[] app() { ua_dialog uad = new ua_dialog(new JFrame(), true); uad.setVisible(true); String username = uad.getUsername(); String server = "sra-ftps.ebi.ac.uk", port = ""; // String server = "localhost", port = ""; try { server = InetAddress.getByName(server).getHostAddress(); } catch (UnknownHostException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } if (username.contains("ega")) { port = "8121"; } else if (username.contains("era")) { port = "8021"; } else if (username.toLowerCase().contains("webin")) { port = "8111"; // New Test Server Port } else if (username.contains("crg")) { server = "84.88.52.70"; // Overwrite standard ENA/EGA server location port = "21"; } uad.dispose(); uad = null; String[] result = { username, server, port, version, build }; return result; } private static String createAndUploadIndex(String nme, FTPClient_alternative client) { String result = null; String upNme = createIndex(nme); if (upNme != null) { try { FileInputStream ins = new FileInputStream(new File(upNme)); String[] r = client.upload(upNme, ins, 0, 0, null, 0, null, null); result = r[0]; } catch (IllegalStateException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPIllegalReplyException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPDataTransferException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } catch (FTPAbortedException ex) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, ex); } } return result; } private static String createIndex(String fle) { try { String inFile = fle; SAMFileReader reader = new SAMFileReader(new File(inFile)); File output = new File(inFile + ".bai"); BAMIndexer indexer = new BAMIndexer(output, reader.getFileHeader()); reader.enableFileSource(true); // create and write the content long count = 0; String str = "."; for (SAMRecord rec : reader) { indexer.processAlignment(rec); } indexer.finish(); reader.close(); return output.getName().toString(); } catch (Throwable th) { Logger.getLogger(NewMain.class.getName()).log(Level.SEVERE, null, th.getMessage()); } return null; } private static void printUsage() { System.out.println(); System.out.println("command line options: "); System.out.println(" []"); System.out.println(" no arguments starts up the GUI login dialog"); System.out.println(" -p -fm -file filename [filename2 filename3 etc.]"); System.out.println(" encrypt-to-file: create encrypted file(s) and MD5 file(s)"); System.out.println(" in the same directory as the original file(s)."); System.out.println(" [-p|-pp <n>] -user <username> -pass <password> -file [filename2 filename3 etc.]"); System.out.println(" encrypt and upload file(s). Order of parameters not important."); System.out.println(" Optional Parameters:"); System.out.println(" -v|-vv: 'verbose' or 'very verbose'"); System.out.println(" -o: 'overwrite' files on the server, if they already exist"); System.out.println(" -udt: use UDT protocol, if available"); System.out.println(" -index: attempt to create an index for BAM files, and upload it as well"); System.out.println(); } private static boolean checkVersion(String[] connect) { if (connect.length >= 1) { // Version Check, if Server responds with a Version if (connect[0].contains("***")) { String expectedversion = connect[0].substring(connect[0].indexOf("***") + 3); double have = Double.parseDouble(NewMain.version); double expected = Double.parseDouble(expectedversion); if (have < expected) { System.out.println( "Client version " + NewMain.version + " out of date (" + expectedversion + ")"); return false; } return true; } } return true; } }