bixo.tools.LengthenUrlsTool.java Source code

Introduction

Here is the source code for bixo.tools.LengthenUrlsTool.java

Source

/*
 * Copyright 2009-2013 Scale Unlimited
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package bixo.tools;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.commons.io.IOUtils;

import bixo.config.BixoPlatform;
import bixo.config.BixoPlatform.Platform;
import bixo.fetcher.BaseFetcher;
import bixo.operations.UrlLengthener;
import bixo.utils.ConfigUtils;
import cascading.flow.Flow;
import cascading.flow.FlowConnector;
import cascading.operation.Debug;
import cascading.pipe.Each;
import cascading.pipe.Pipe;
import cascading.scheme.local.TextLine;
import cascading.tap.SinkMode;
import cascading.tap.SinkTap;
import cascading.tap.Tap;
import cascading.tuple.Fields;

import com.scaleunlimited.cascading.BasePath;
import com.scaleunlimited.cascading.NullSinkTap;

public class LengthenUrlsTool {

    private static String readInputLine() throws IOException {
        InputStreamReader isr = new InputStreamReader(System.in);
        BufferedReader br = new BufferedReader(isr);

        try {
            return br.readLine();
        } finally {
            // TODO KKr - will this actually close System.in?
            // Should I reuse this buffered reader? Check out password masking code.
            // br.close();
        }
    }

    /**
     * @param args - URL to fetch, or path to file of URLs
     */
    @SuppressWarnings("rawtypes")
    public static void main(String[] args) {
        try {
            String url = null;
            if (args.length == 0) {
                System.out.print("URL to lengthen: ");
                url = readInputLine();
                if (url.length() == 0) {
                    System.exit(0);
                }

                if (!url.startsWith("http://")) {
                    url = "http://" + url;
                }
            } else if (args.length != 1) {
                System.out.print("A single URL or filename parameter is allowed");
                System.exit(0);
            } else {
                url = args[0];
            }

            String filename;
            if (!url.startsWith("http://")) {
                // It's a path to a file of URLs
                filename = url;
            } else {
                // We have a URL that we need to write to a temp file.
                File tempFile = File.createTempFile("LengthenUrlsTool", "txt");
                filename = tempFile.getAbsolutePath();
                FileWriter fw = new FileWriter(tempFile);
                IOUtils.write(url, fw);
                fw.close();
            }

            System.setProperty("bixo.root.level", "TRACE");
            // Uncomment this to see the wire log for HttpClient
            // System.setProperty("bixo.http.level", "DEBUG");

            BaseFetcher fetcher = UrlLengthener.makeFetcher(10, ConfigUtils.BIXO_TOOL_AGENT);

            Pipe pipe = new Pipe("urls");
            pipe = new Each(pipe, new UrlLengthener(fetcher));
            pipe = new Each(pipe, new Debug());

            BixoPlatform platform = new BixoPlatform(LengthenUrlsTool.class, Platform.Local);
            BasePath filePath = platform.makePath(filename);
            TextLine textLineLocalScheme = new TextLine(new Fields("url"));
            Tap sourceTap = platform.makeTap(textLineLocalScheme, filePath, SinkMode.KEEP);
            SinkTap sinkTap = new NullSinkTap(new Fields("url"));

            FlowConnector flowConnector = platform.makeFlowConnector();
            Flow flow = flowConnector.connect(sourceTap, sinkTap, pipe);

            flow.complete();
        } catch (Exception e) {
            System.err.println("Exception running tool: " + e.getMessage());
            e.printStackTrace(System.err);
            System.exit(-1);
        }
    }

}