Example usage for java.io FileInputStream close

List of usage examples for java.io FileInputStream close

  1. HOME
  2. Java
  3. java
  4. java.io.*
  5. FileInputStream
  6. close

Introduction

In this page you can find the example usage for java.io FileInputStream close.

Prototype

public void close() throws IOException

Source Link

Document

Closes this file input stream and releases any system resources associated with the stream.

Usage

From source file:com.enjoyxstudy.selenium.htmlsuite.MultiHTMLSuiteRunner.java

/**
 * @param args/*from   w w w . j  a  v a2  s  .  c  o m*/
 * @throws Exception
 */
public static void main(String[] args) throws Exception {

    String propertyFile = "htmlSuite.properties"; // default
    if (args.length != 0) {
        propertyFile = args[0];
    }
    Properties properties = new Properties();
    FileInputStream inputStream = new FileInputStream(propertyFile);
    try {
        properties.load(inputStream);
    } finally {
        inputStream.close();
    }

    MultiHTMLSuiteRunner runner = execute(properties);

    System.exit(runner.getResult() ? 0 : 1);
}

From source file:MainClass.java

public static void main(String[] args) throws Exception {
    createFile();/*from   w w w.  jav a 2s  .  c  o  m*/

    File aFile = new File("C:/primes.bin");
    FileInputStream inFile = new FileInputStream(aFile);

    FileChannel inChannel = inFile.getChannel();

    final int PRIMESREQUIRED = 10;
    ByteBuffer buf = ByteBuffer.allocate(8 * PRIMESREQUIRED);

    long[] primes = new long[PRIMESREQUIRED];
    int index = 0; // Position for a prime in the file

    // Count of primes in the file
    final int PRIMECOUNT = (int) inChannel.size() / 8;

    // Read the number of random primes required
    for (int i = 0; i < PRIMESREQUIRED; i++) {
        index = 8 * (int) (PRIMECOUNT * Math.random());
        inChannel.read(buf, index); // Read the value
        buf.flip();
        primes[i] = buf.getLong(); // Save it in the array
        buf.clear();
    }

    for (long prime : primes) {
        System.out.printf("%12d", prime);
    }
    inFile.close(); // Close the file and the channel

}

From source file:com.sight.facialrecog.util.ImageManipulation.java

/**
 * @param args/* w  w  w  .j a va 2  s  . c  o m*/
 */
public static void main(String[] args) {
    File file = new File("/home/sight/Pictures/imagetest/close.jpg");

    try {
        /*
         * Reading an Image file from file system
         */
        FileInputStream imageInFile = new FileInputStream(file);
        byte imageData[] = new byte[(int) file.length()];
        imageInFile.read(imageData);

        /*
         * Converting Image byte array into Base64 String
         */
        String imageDataString = encodeImage(imageData);
        System.out.println(imageDataString);

        /*
         * Converting a Base64 String into Image byte array
         */
        byte[] imageByteArray = decodeImage(imageDataString);

        /*
         * Write an image byte array into file system
         */
        FileOutputStream imageOutFile = new FileOutputStream(
                "/home/sight/Pictures/imagetest/close-after-convert.jpg");
        imageOutFile.write(imageByteArray);

        imageInFile.close();
        imageOutFile.close();

        System.out.println("Image Successfully Manipulated!");
    } catch (FileNotFoundException e) {
        System.out.println("Image not found" + e);
    } catch (IOException ioe) {
        System.out.println("Exception while reading the Image " + ioe);
    }

}

From source file:net.iiit.siel.analysis.lang.NGramProfile.java

/**
 * main method used for testing only/*from w  ww  .  j a v a 2 s  . com*/
 * 
 * @param args
 */
public static void main(String args[]) {

    String usage = "Usage: NGramProfile " + "[-create profilename filename encoding] "
            + "[-similarity file1 file2] " + "[-score profile-name filename encoding]";
    int command = 0;

    final int CREATE = 1;
    final int SIMILARITY = 2;
    final int SCORE = 3;

    String profilename = "";
    String filename = "";
    String filename2 = "";
    String encoding = "";

    if (args.length == 0) {
        System.err.println(usage);
        System.exit(-1);
    }

    for (int i = 0; i < args.length; i++) { // parse command line
        if (args[i].equals("-create")) { // found -create option
            command = CREATE;
            profilename = args[++i];
            filename = args[++i];
            encoding = args[++i];
        }

        if (args[i].equals("-similarity")) { // found -similarity option
            command = SIMILARITY;
            filename = args[++i];
            filename2 = args[++i];
            encoding = args[++i];
        }

        if (args[i].equals("-score")) { // found -Score option
            command = SCORE;
            profilename = args[++i];
            filename = args[++i];
            encoding = args[++i];
        }
    }

    try {

        switch (command) {

        case CREATE:

            File f = new File(filename);
            FileInputStream fis = new FileInputStream(f);
            NGramProfile newProfile = NGramProfile.create(profilename, fis, encoding);
            fis.close();
            f = new File(profilename + "." + FILE_EXTENSION);
            FileOutputStream fos = new FileOutputStream(f);
            newProfile.save(fos);
            System.out.println("new profile " + profilename + "." + FILE_EXTENSION + " was created.");
            break;

        case SIMILARITY:

            f = new File(filename);
            fis = new FileInputStream(f);
            newProfile = NGramProfile.create(filename, fis, encoding);
            newProfile.normalize();

            f = new File(filename2);
            fis = new FileInputStream(f);
            NGramProfile newProfile2 = NGramProfile.create(filename2, fis, encoding);
            newProfile2.normalize();
            System.out.println("Similarity is " + newProfile.getSimilarity(newProfile2));
            break;

        case SCORE:
            f = new File(filename);
            fis = new FileInputStream(f);
            newProfile = NGramProfile.create(filename, fis, encoding);

            f = new File(profilename + "." + FILE_EXTENSION);
            fis = new FileInputStream(f);
            NGramProfile compare = new NGramProfile(profilename, DEFAULT_MIN_NGRAM_LENGTH,
                    DEFAULT_MAX_NGRAM_LENGTH);
            compare.load(fis);
            System.out.println("Score is " + compare.getSimilarity(newProfile));
            break;

        }

    } catch (Exception e) {
        if (LOG.isErrorEnabled()) {
            LOG.error("Caught an exception:" + e);
        }
    }
}

From source file:com.interflora.ftp.SampleFTPS.java

public static final void main(String[] args) {
    boolean storeFile = true, binaryTransfer = false, error = false, listFiles = false, listNames = false,
            hidden = false;//from  www  .  ja v  a  2  s .  c om
    boolean localActive = false, useEpsvWithIPv4 = false, feat = false, printHash = false;
    boolean mlst = false, mlsd = false;
    boolean lenient = false;
    long keepAliveTimeout = -1;
    int controlKeepAliveReplyTimeout = -1;
    int minParams = 5; // listings require 3 params
    String protocol = "TLS,false"; // SSL protocol
    String doCommand = null;
    String trustmgr = "all";
    String proxyHost = null;
    int proxyPort = 80;
    String proxyUser = null;
    String proxyPassword = null;
    //sample : rose-misc01.hyd.ftd.untd.com rosedev test123 /TestSftp  C:/Users/skarin/Desktop/RosePaths.txt 
    //172.16.160.70 demand quevfoc5 /TestSftp C:/Users/skarin/Desktop/RosePaths.txt 
    int base = 0;
    for (base = 0; base < args.length; base++) {
        if (args[base].equals("-s")) {
            storeFile = true;
        } else if (args[base].equals("-a")) {
            localActive = true;
        } else if (args[base].equals("-b")) {
            binaryTransfer = true;
        } else if (args[base].equals("-c")) {
            doCommand = args[++base];
            minParams = 3;
        } else if (args[base].equals("-d")) {
            mlsd = true;
            minParams = 3;
        } else if (args[base].equals("-e")) {
            useEpsvWithIPv4 = true;
        } else if (args[base].equals("-f")) {
            feat = true;
            minParams = 3;
        } else if (args[base].equals("-h")) {
            hidden = true;
        } else if (args[base].equals("-k")) {
            keepAliveTimeout = Long.parseLong(args[++base]);
        } else if (args[base].equals("-l")) {
            listFiles = true;
            minParams = 3;
        } else if (args[base].equals("-L")) {
            lenient = true;
        } else if (args[base].equals("-n")) {
            listNames = true;
            minParams = 3;
        } else if (args[base].equals("-p")) {
            protocol = args[++base];
        } else if (args[base].equals("-t")) {
            mlst = true;
            minParams = 3;
        } else if (args[base].equals("-w")) {
            controlKeepAliveReplyTimeout = Integer.parseInt(args[++base]);
        } else if (args[base].equals("-T")) {
            trustmgr = args[++base];
        } else if (args[base].equals("-PrH")) {
            proxyHost = args[++base];
            String parts[] = proxyHost.split(":");
            if (parts.length == 2) {
                proxyHost = parts[0];
                proxyPort = Integer.parseInt(parts[1]);
            }
        } else if (args[base].equals("-PrU")) {
            proxyUser = args[++base];
        } else if (args[base].equals("-PrP")) {
            proxyPassword = args[++base];
        } else if (args[base].equals("-#")) {
            printHash = true;
        } else {
            break;
        }
    }

    int remain = args.length - base;
    if (remain < minParams) // server, user, pass, remote, local [protocol]
    {
        System.err.println(USAGE);
        System.exit(1);
    }

    String server = args[base++];
    int port = 0;
    String parts[] = server.split(":");
    if (parts.length == 2) {
        server = parts[0];
        port = Integer.parseInt(parts[1]);
    }
    String username = args[base++];
    String password = args[base++];

    String remote = null;
    if (args.length - base > 0) {
        remote = args[base++];
    }

    String local = null;
    if (args.length - base > 0) {
        local = args[base++];
    }

    final FTPClient ftp;
    if (protocol == null) {
        if (proxyHost != null) {
            System.out.println("Using HTTP proxy server: " + proxyHost);
            ftp = new FTPHTTPClient(proxyHost, proxyPort, proxyUser, proxyPassword);
        } else {
            ftp = new FTPClient();
        }
    } else {
        FTPSClient ftps;
        if (protocol.equals("true")) {
            ftps = new FTPSClient(true);
        } else if (protocol.equals("false")) {
            ftps = new FTPSClient(false);
        } else {
            String prot[] = protocol.split(",");
            if (prot.length == 1) { // Just protocol
                ftps = new FTPSClient(protocol);
            } else { // protocol,true|false
                ftps = new FTPSClient(prot[0], Boolean.parseBoolean(prot[1]));
            }
        }
        ftp = ftps;
        if ("all".equals(trustmgr)) {
            ftps.setTrustManager(TrustManagerUtils.getAcceptAllTrustManager());
        } else if ("valid".equals(trustmgr)) {
            ftps.setTrustManager(TrustManagerUtils.getValidateServerCertificateTrustManager());
        } else if ("none".equals(trustmgr)) {
            ftps.setTrustManager(null);
        }
        /* try{
         //ftps.setFileType(FTP.ASCII_FILE_TYPE);
         ftps.execPBSZ(0); // Set protection buffer size
         ftps.execPROT("C"); // Set data channel protection to private
         }catch(IOException ex ){ex.printStackTrace();};
        */
    }
    // System.out.println("After I/O Exception");

    if (printHash) {
        ftp.setCopyStreamListener(createListener());
    }
    if (keepAliveTimeout >= 0) {
        ftp.setControlKeepAliveTimeout(keepAliveTimeout);
    }
    if (controlKeepAliveReplyTimeout >= 0) {
        ftp.setControlKeepAliveReplyTimeout(controlKeepAliveReplyTimeout);
    }
    ftp.setListHiddenFiles(hidden);

    // suppress login details
    ftp.addProtocolCommandListener(new PrintCommandListener(new PrintWriter(System.out), true));

    try {
        int reply;
        if (port > 0) {
            ftp.connect(server, port);
        } else {
            System.out.println("Trying to Connect to the server \t" + server);
            ftp.connect(server);
        }
        System.out.println("Connected to " + server + " on " + (port > 0 ? port : ftp.getDefaultPort()));

        // After connection attempt, you should check the reply code to verify
        // success.
        reply = ftp.getReplyCode();

        if (!FTPReply.isPositiveCompletion(reply)) {
            ftp.disconnect();
            System.err.println("FTP server refused connection.");
            System.exit(1);
        }
    } catch (IOException e) {
        if (ftp.isConnected()) {
            try {
                ftp.disconnect();
            } catch (IOException f) {
                // do nothing
            }
        }
        System.err.println("Could not connect to server.");
        e.printStackTrace();
        System.exit(1);
    }

    __main: try {
        if (!ftp.login(username, password)) {
            ftp.logout();
            error = true;
            break __main;
        }

        System.out.println("Remote system is " + ftp.getSystemType());

        if (binaryTransfer) {
            ftp.setFileType(FTP.BINARY_FILE_TYPE);
        }

        // Use passive mode as default because most of us are
        // behind firewalls these days.
        if (localActive) {
            ftp.enterLocalActiveMode();
        } else {
            ftp.enterLocalPassiveMode();
        }

        ftp.setUseEPSVwithIPv4(useEpsvWithIPv4);

        if (storeFile) {
            //InputStream input;
            File fname = new File(local);

            FileInputStream input = new FileInputStream(fname);
            String localfilename = fname.getName();
            System.out.println("Local file name " + localfilename);

            ftp.storeFile(remote + "/test/" + localfilename, input);

            input.close();
        } else if (listFiles) {
            if (lenient) {
                FTPClientConfig config = new FTPClientConfig();
                config.setLenientFutureDates(true);
                ftp.configure(config);
            }

            for (FTPFile f : ftp.listFiles(remote)) {
                System.out.println(f.getRawListing());
                System.out.println(f.toFormattedString());
            }
        } else if (mlsd) {
            for (FTPFile f : ftp.mlistDir(remote)) {
                System.out.println(f.getRawListing());
                System.out.println(f.toFormattedString());
            }
        } else if (mlst) {
            FTPFile f = ftp.mlistFile(remote);
            if (f != null) {
                System.out.println(f.toFormattedString());
            }
        } else if (listNames) {
            for (String s : ftp.listNames(remote)) {
                System.out.println(s);
            }
        } else if (feat) {
            // boolean feature check
            if (remote != null) { // See if the command is present
                if (ftp.hasFeature(remote)) {
                    System.out.println("Has feature: " + remote);
                } else {
                    if (FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
                        System.out.println("FEAT " + remote + " was not detected");
                    } else {
                        System.out.println("Command failed: " + ftp.getReplyString());
                    }
                }

                // Strings feature check
                String[] features = ftp.featureValues(remote);
                if (features != null) {
                    for (String f : features) {
                        System.out.println("FEAT " + remote + "=" + f + ".");
                    }
                } else {
                    if (FTPReply.isPositiveCompletion(ftp.getReplyCode())) {
                        System.out.println("FEAT " + remote + " is not present");
                    } else {
                        System.out.println("Command failed: " + ftp.getReplyString());
                    }
                }
            } else {
                if (ftp.features()) {
                    //                           Command listener has already printed the output
                } else {
                    System.out.println("Failed: " + ftp.getReplyString());
                }
            }
        } else if (doCommand != null) {
            if (ftp.doCommand(doCommand, remote)) {
                //                     Command listener has already printed the output
                //                       for(String s : ftp.getReplyStrings()) {
                //                           System.out.println(s);
                //                       }
            } else {
                System.out.println("Failed: " + ftp.getReplyString());
            }
        } else {
            OutputStream output;

            output = new FileOutputStream(local);

            ftp.retrieveFile(remote, output);

            output.close();
        }

        ftp.noop(); // check that control connection is working OK

        ftp.logout();
    } catch (FTPConnectionClosedException e) {
        error = true;
        System.err.println("Server closed connection.");
        e.printStackTrace();
    } catch (IOException e) {
        error = true;
        e.printStackTrace();
    } finally {
        if (ftp.isConnected()) {
            try {
                ftp.disconnect();
            } catch (IOException f) {
                // do nothing
            }
        }
    }

    System.exit(error ? 1 : 0);
}

From source file:cc.twittertools.index.IndexStatuses.java

@SuppressWarnings("static-access")
public static void main(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(new Option(HELP_OPTION, "show help"));
    options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment"));
    options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors"));

    options.addOption(OptionBuilder.withArgName("dir").hasArg().withDescription("source collection directory")
            .create(COLLECTION_OPTION));
    options.addOption(/*from  w w  w. ja v  a2 s .  c o m*/
            OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION));
    options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids")
            .create(DELETES_OPTION));
    options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        System.exit(-1);
    }

    if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION)
            || !cmdline.hasOption(INDEX_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(IndexStatuses.class.getName(), options);
        System.exit(-1);
    }

    String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION);
    String indexPath = cmdline.getOptionValue(INDEX_OPTION);

    final FieldType textOptions = new FieldType();
    textOptions.setIndexed(true);
    textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
    textOptions.setStored(true);
    textOptions.setTokenized(true);
    if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) {
        textOptions.setStoreTermVectors(true);
    }

    LOG.info("collection: " + collectionPath);
    LOG.info("index: " + indexPath);

    LongOpenHashSet deletes = null;
    if (cmdline.hasOption(DELETES_OPTION)) {
        deletes = new LongOpenHashSet();
        File deletesFile = new File(cmdline.getOptionValue(DELETES_OPTION));
        if (!deletesFile.exists()) {
            System.err.println("Error: " + deletesFile + " does not exist!");
            System.exit(-1);
        }
        LOG.info("Reading deletes from " + deletesFile);

        FileInputStream fin = new FileInputStream(deletesFile);
        byte[] ignoreBytes = new byte[2];
        fin.read(ignoreBytes); // "B", "Z" bytes from commandline tools
        BufferedReader br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fin)));

        String s;
        while ((s = br.readLine()) != null) {
            if (s.contains("\t")) {
                deletes.add(Long.parseLong(s.split("\t")[0]));
            } else {
                deletes.add(Long.parseLong(s));
            }
        }
        br.close();
        fin.close();
        LOG.info("Read " + deletes.size() + " tweetids from deletes file.");
    }

    long maxId = Long.MAX_VALUE;
    if (cmdline.hasOption(MAX_ID_OPTION)) {
        maxId = Long.parseLong(cmdline.getOptionValue(MAX_ID_OPTION));
        LOG.info("index: " + maxId);
    }

    long startTime = System.currentTimeMillis();
    File file = new File(collectionPath);
    if (!file.exists()) {
        System.err.println("Error: " + file + " does not exist!");
        System.exit(-1);
    }

    StatusStream stream = new JsonStatusCorpusReader(file);

    Directory dir = FSDirectory.open(new File(indexPath));
    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, IndexStatuses.ANALYZER);
    config.setOpenMode(OpenMode.CREATE);

    IndexWriter writer = new IndexWriter(dir, config);
    int cnt = 0;
    Status status;
    try {
        while ((status = stream.next()) != null) {
            if (status.getText() == null) {
                continue;
            }

            // Skip deletes tweetids.
            if (deletes != null && deletes.contains(status.getId())) {
                continue;
            }

            if (status.getId() > maxId) {
                continue;
            }

            cnt++;
            Document doc = new Document();
            doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES));
            doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES));
            doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES));

            doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions));

            doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES));
            doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES));
            doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES));

            long inReplyToStatusId = status.getInReplyToStatusId();
            if (inReplyToStatusId > 0) {
                doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId,
                        Field.Store.YES));
                doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(),
                        Field.Store.YES));
            }

            String lang = status.getLang();
            if (!lang.equals("unknown")) {
                doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES));
            }

            long retweetStatusId = status.getRetweetedStatusId();
            if (retweetStatusId > 0) {
                doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES));
                doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(),
                        Field.Store.YES));
                doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES));
                if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) {
                    LOG.warn("Error parsing retweet fields of " + status.getId());
                }
            }

            writer.addDocument(doc);
            if (cnt % 100000 == 0) {
                LOG.info(cnt + " statuses indexed");
            }
        }

        LOG.info(String.format("Total of %s statuses added", cnt));

        if (cmdline.hasOption(OPTIMIZE_OPTION)) {
            LOG.info("Merging segments...");
            writer.forceMerge(1);
            LOG.info("Done!");
        }

        LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms");
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        writer.close();
        dir.close();
        stream.close();
    }
}

From source file:io.anserini.index.IndexTweets.java

@SuppressWarnings("static-access")
public static void main(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(new Option(HELP_OPTION, "show help"));
    options.addOption(new Option(OPTIMIZE_OPTION, "merge indexes into a single segment"));
    options.addOption(new Option(STORE_TERM_VECTORS_OPTION, "store term vectors"));

    options.addOption(OptionBuilder.withArgName("dir").hasArg().withDescription("source collection directory")
            .create(COLLECTION_OPTION));
    options.addOption(//w  w  w  .j a v a2 s .  c o  m
            OptionBuilder.withArgName("dir").hasArg().withDescription("index location").create(INDEX_OPTION));
    options.addOption(OptionBuilder.withArgName("file").hasArg().withDescription("file with deleted tweetids")
            .create(DELETES_OPTION));
    options.addOption(OptionBuilder.withArgName("id").hasArg().withDescription("max id").create(MAX_ID_OPTION));

    CommandLine cmdline = null;
    CommandLineParser parser = new GnuParser();
    try {
        cmdline = parser.parse(options, args);
    } catch (ParseException exp) {
        System.err.println("Error parsing command line: " + exp.getMessage());
        System.exit(-1);
    }

    if (cmdline.hasOption(HELP_OPTION) || !cmdline.hasOption(COLLECTION_OPTION)
            || !cmdline.hasOption(INDEX_OPTION)) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp(IndexTweets.class.getName(), options);
        System.exit(-1);
    }

    String collectionPath = cmdline.getOptionValue(COLLECTION_OPTION);
    String indexPath = cmdline.getOptionValue(INDEX_OPTION);

    final FieldType textOptions = new FieldType();
    textOptions.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
    textOptions.setStored(true);
    textOptions.setTokenized(true);
    if (cmdline.hasOption(STORE_TERM_VECTORS_OPTION)) {
        textOptions.setStoreTermVectors(true);
    }

    LOG.info("collection: " + collectionPath);
    LOG.info("index: " + indexPath);

    LongOpenHashSet deletes = null;
    if (cmdline.hasOption(DELETES_OPTION)) {
        deletes = new LongOpenHashSet();
        File deletesFile = new File(cmdline.getOptionValue(DELETES_OPTION));
        if (!deletesFile.exists()) {
            System.err.println("Error: " + deletesFile + " does not exist!");
            System.exit(-1);
        }
        LOG.info("Reading deletes from " + deletesFile);

        FileInputStream fin = new FileInputStream(deletesFile);
        byte[] ignoreBytes = new byte[2];
        fin.read(ignoreBytes); // "B", "Z" bytes from commandline tools
        BufferedReader br = new BufferedReader(new InputStreamReader(new CBZip2InputStream(fin)));

        String s;
        while ((s = br.readLine()) != null) {
            if (s.contains("\t")) {
                deletes.add(Long.parseLong(s.split("\t")[0]));
            } else {
                deletes.add(Long.parseLong(s));
            }
        }
        br.close();
        fin.close();
        LOG.info("Read " + deletes.size() + " tweetids from deletes file.");
    }

    long maxId = Long.MAX_VALUE;
    if (cmdline.hasOption(MAX_ID_OPTION)) {
        maxId = Long.parseLong(cmdline.getOptionValue(MAX_ID_OPTION));
        LOG.info("index: " + maxId);
    }

    long startTime = System.currentTimeMillis();
    File file = new File(collectionPath);
    if (!file.exists()) {
        System.err.println("Error: " + file + " does not exist!");
        System.exit(-1);
    }

    StatusStream stream = new JsonStatusCorpusReader(file);

    Directory dir = FSDirectory.open(Paths.get(indexPath));
    final IndexWriterConfig config = new IndexWriterConfig(ANALYZER);

    config.setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    IndexWriter writer = new IndexWriter(dir, config);
    int cnt = 0;
    Status status;
    try {
        while ((status = stream.next()) != null) {
            if (status.getText() == null) {
                continue;
            }

            // Skip deletes tweetids.
            if (deletes != null && deletes.contains(status.getId())) {
                continue;
            }

            if (status.getId() > maxId) {
                continue;
            }

            cnt++;
            Document doc = new Document();
            doc.add(new LongPoint(StatusField.ID.name, status.getId()));
            doc.add(new StoredField(StatusField.ID.name, status.getId()));
            doc.add(new LongPoint(StatusField.EPOCH.name, status.getEpoch()));
            doc.add(new StoredField(StatusField.EPOCH.name, status.getEpoch()));
            doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES));

            doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions));

            doc.add(new IntPoint(StatusField.FRIENDS_COUNT.name, status.getFollowersCount()));
            doc.add(new StoredField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount()));
            doc.add(new IntPoint(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount()));
            doc.add(new StoredField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount()));
            doc.add(new IntPoint(StatusField.STATUSES_COUNT.name, status.getStatusesCount()));
            doc.add(new StoredField(StatusField.STATUSES_COUNT.name, status.getStatusesCount()));

            long inReplyToStatusId = status.getInReplyToStatusId();
            if (inReplyToStatusId > 0) {
                doc.add(new LongPoint(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId));
                doc.add(new StoredField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId));
                doc.add(new LongPoint(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId()));
                doc.add(new StoredField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId()));
            }

            String lang = status.getLang();
            if (!lang.equals("unknown")) {
                doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES));
            }

            long retweetStatusId = status.getRetweetedStatusId();
            if (retweetStatusId > 0) {
                doc.add(new LongPoint(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId));
                doc.add(new StoredField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId));
                doc.add(new LongPoint(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId()));
                doc.add(new StoredField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId()));
                doc.add(new IntPoint(StatusField.RETWEET_COUNT.name, status.getRetweetCount()));
                doc.add(new StoredField(StatusField.RETWEET_COUNT.name, status.getRetweetCount()));
                if (status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) {
                    LOG.warn("Error parsing retweet fields of " + status.getId());
                }
            }

            writer.addDocument(doc);
            if (cnt % 100000 == 0) {
                LOG.info(cnt + " statuses indexed");
            }
        }

        LOG.info(String.format("Total of %s statuses added", cnt));

        if (cmdline.hasOption(OPTIMIZE_OPTION)) {
            LOG.info("Merging segments...");
            writer.forceMerge(1);
            LOG.info("Done!");
        }

        LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms");
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        writer.close();
        dir.close();
        stream.close();
    }
}

From source file:MainClass.java

public static void main(String[] args) {
    File aFile = new File("primes.txt");
    FileInputStream inFile = null;
    try {// w ww.j a v  a2  s  .c  om
        inFile = new FileInputStream(aFile);
    } catch (FileNotFoundException e) {
        e.printStackTrace(System.err);
    }
    FileChannel inChannel = inFile.getChannel();
    try {
        ByteBuffer lengthBuf = ByteBuffer.allocate(8);
        while (true) {
            if (inChannel.read(lengthBuf) == -1) {
                break;
            }
            lengthBuf.flip();
            int strLength = (int) lengthBuf.getDouble();
            ByteBuffer buf = ByteBuffer.allocate(2 * strLength + 8);
            if (inChannel.read(buf) == -1) {
                break;
            }
            buf.flip();
            byte[] strChars = new byte[2 * strLength];
            buf.get(strChars);
            System.out.println(strLength);
            System.out.println(ByteBuffer.wrap(strChars).asCharBuffer());
            System.out.println(buf.getLong());
            lengthBuf.clear();
        }
        inFile.close();
    } catch (IOException e) {
        e.printStackTrace(System.err);
    }
}

From source file:Main.java

public static void main(String[] args) throws Exception {
    File aFile = new File("primes.txt");
    FileInputStream inFile = new FileInputStream(aFile);
    FileChannel inChannel = inFile.getChannel();
    ByteBuffer buf = ByteBuffer.allocateDirect(1024);
    buf.position(buf.limit());/*w w  w .jav a2 s . com*/
    while (true) {
        if (buf.remaining() < 8) {
            if (inChannel.read(buf.compact()) == -1) {
                break;
            }
            buf.flip();
        }
        int strLength = (int) buf.getDouble();
        if (buf.remaining() < 2 * strLength) {
            if (inChannel.read(buf.compact()) == -1) {
                break;
            }
            buf.flip();
        }
        byte[] strChars = new byte[2 * strLength];
        buf.get(strChars);
        if (buf.remaining() < 8) {
            if (inChannel.read(buf.compact()) == -1) {
                break;
            }
            buf.flip();
        }
        System.out.println(strLength);
        System.out.println(ByteBuffer.wrap(strChars).asCharBuffer());
        System.out.println(buf.getLong());
    }
    inFile.close();
}

From source file:MainClass.java

public static void main(String[] args) {
    File aFile = new File("data.dat");
    FileInputStream inFile = null;

    try {/*from ww w.  j a v  a 2  s .  c om*/
        inFile = new FileInputStream(aFile);
    } catch (FileNotFoundException e) {
        e.printStackTrace(System.err);
        System.exit(1);
    }

    FileChannel inChannel = inFile.getChannel();
    final int COUNT = 6;
    ByteBuffer buf = ByteBuffer.allocate(8 * COUNT);
    long[] data = new long[COUNT];
    try {
        int pos = 0;
        while (inChannel.read(buf) != -1) {
            try {
                ((ByteBuffer) (buf.flip())).asLongBuffer().get(data);
                pos = data.length;
            } catch (BufferUnderflowException e) {
                LongBuffer longBuf = buf.asLongBuffer();
                pos = longBuf.remaining();
                longBuf.get(data, 0, pos);
            }
            System.out.println();
            for (int i = 0; i < pos; i++) {
                System.out.printf("%10d", data[i]);
            }
            buf.clear();
        }
        System.out.println("\nEOF reached.");
        inFile.close();
    } catch (IOException e) {
        e.printStackTrace(System.err);
        System.exit(1);
    }
}