Example usage for org.apache.hadoop.io Text set

List of usage examples for org.apache.hadoop.io Text set

  1. HOME
  2. Java
  3. org.apache.hadoop
  4. org.apache.hadoop.io.*
  5. Text
  6. set

Introduction

In this page you can find the example usage for org.apache.hadoop.io Text set.

Prototype

public void set(Text other)

Source Link

Document

copy a text.

Usage

From source file:edu.umd.windmemory.BooleanRetrievalCompressed.java

License:Apache License

private ArrayListWritable<VIntWritable> fetchPostings(String term) throws IOException {
    Text key = new Text();
    ArrayListWritable<VIntWritable> value = new ArrayListWritable<VIntWritable>();

    key.set(term);
    index.get(key, value);/*from   w  w  w .j  a v  a 2 s.  co  m*/

    return value;
}

From source file:edu.umn.cs.spatialHadoop.CommandLineArguments.java

License:Apache License

/**
 * /*from   www  .j a va 2s.  co m*/
 * @param autodetect - Automatically detect shape type from input file
 *   if shape is not explicitly set by user
 * @return
 */
public Shape getShape(boolean autodetect) {
    String shapeTypeStr = get("shape");
    final Text shapeType = new Text();
    if (shapeTypeStr != null)
        shapeType.set(shapeTypeStr.toLowerCase().getBytes());

    if (autodetect && shapeType.getLength() == 0 && getPath() != null) {
        // Shape type not found in parameters. Try to infer from a line in input
        // file
        Path in_file = getPath();
        try {
            Sampler.sampleLocal(in_file.getFileSystem(new Configuration()), in_file, 1, 0,
                    new ResultCollector<Text2>() {
                        @Override
                        public void collect(Text2 value) {
                            String val = value.toString();
                            String[] parts = val.split(",");
                            if (parts.length == 2) {
                                shapeType.set("point".getBytes());
                            } else if (parts.length == 4) {
                                shapeType.set("rect".getBytes());
                            } else if (parts.length > 4) {
                                shapeType.set("tiger".getBytes());
                            }
                        }
                    }, new Text2(), new Text2());
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    Shape stockShape = null;
    if (shapeType.toString().startsWith("rect")) {
        stockShape = new Rectangle();
    } else if (shapeType.toString().startsWith("point")) {
        stockShape = new Point();
    } else if (shapeType.toString().startsWith("tiger")) {
        stockShape = new TigerShape();
    } else if (shapeType.toString().startsWith("poly")) {
        stockShape = new Polygon();
    } else if (shapeType.toString().startsWith("ogc")) {
        stockShape = new OGCShape();
    } else if (shapeType.toString().startsWith("nasa")) {
        stockShape = new NASAPoint();
    } else if (shapeTypeStr != null) {
        // Use the shapeType as a class name and try to instantiate it dynamically
        try {
            Class<? extends Shape> shapeClass = Class.forName(shapeTypeStr).asSubclass(Shape.class);
            stockShape = shapeClass.newInstance();
        } catch (ClassNotFoundException e) {
        } catch (InstantiationException e) {
        } catch (IllegalAccessException e) {
        }
    }
    if (stockShape == null)
        LOG.warn("unknown shape type: " + shapeTypeStr);

    return stockShape;
}

From source file:edu.umn.cs.spatialHadoop.CommandLineArguments.java

License:Apache License

public Shape getOutputShape() {
    String shapeTypeStr = get("outshape");
    if (shapeTypeStr == null)
        shapeTypeStr = get("outputshape");
    if (shapeTypeStr == null)
        shapeTypeStr = get("output_shape");
    final Text shapeType = new Text();
    if (shapeTypeStr != null)
        shapeType.set(shapeTypeStr.toLowerCase().getBytes());

    Shape stockShape = null;/*  ww  w . j  a va 2  s.c om*/
    if (shapeType.toString().startsWith("rect")) {
        stockShape = new Rectangle();
    } else if (shapeType.toString().startsWith("point")) {
        stockShape = new Point();
    } else if (shapeType.toString().startsWith("tiger")) {
        stockShape = new TigerShape();
    } else if (shapeType.toString().startsWith("poly")) {
        stockShape = new Polygon();
    } else if (shapeType.toString().startsWith("ogc")) {
        stockShape = new OGCShape();
    } else if (shapeTypeStr != null) {
        // Use the shapeType as a class name and try to instantiate it dynamically
        try {
            Class<? extends Shape> shapeClass = Class.forName(shapeTypeStr).asSubclass(Shape.class);
            stockShape = shapeClass.newInstance();
        } catch (ClassNotFoundException e) {
        } catch (InstantiationException e) {
        } catch (IllegalAccessException e) {
        }
    }
    if (stockShape == null)
        LOG.warn("unknown shape type: " + shapeTypeStr);

    return stockShape;
}

From source file:edu.umn.cs.spatialHadoop.TigerShape.java

License:Open Source License

@Override
public Text toText(Text text) {
    text.set(originalText);
    return text;
}

From source file:edu.yale.cs.hadoopdb.sms.connector.SMSRecordReader.java

License:Apache License

/**
 * Retrieves each row from the result set, serializes it 
 * using {@link ParseSchema} and increments the number of rows
 * read in./*from   w w w . j  a  v  a 2  s.c om*/
 * @return false if no more rows exist.
 */
@Override
public boolean next(LongWritable key, Text value) throws IOException {
    try {
        if (!results.next())
            return false;
        key.set(pos);
        value.set(parseResults());
        pos++;
    } catch (SQLException e) {
        throw new IOException(e);
    }
    return true;
}

From source file:eu.scape_project.tb.lsdr.seqfileutility.SequenceFileWriter.java

License:Apache License

private void writeTextLines(File file) throws FileNotFoundException, IOException {
    BufferedReader buffer = new BufferedReader(new FileReader(file));
    String line;//w ww . j av  a  2  s . co  m
    Text key = new Text();
    key.set(file.getAbsolutePath());
    Text value = new Text();

    while ((line = buffer.readLine()) != null) {
        value.set(line);
        writer.append(key, value);
        logger.info(this.getId() + ": " + value);
        linecount++;
    }
    buffer.close();
}

From source file:eu.scape_project.tb.lsdr.seqfileutility.SequenceFileWriter.java

License:Apache License

private void writeFileContent(File file) throws IOException {
    long fileLength = file.length();
    if (fileLength <= Integer.MAX_VALUE) {
        Text key = new Text();
        String filePath = file.getAbsolutePath();
        String keyPath = FilenameUtils.separatorsToUnix(filePath);
        key.set(keyPath);

        FileInputStream fis = new FileInputStream(file);
        byte[] byteArray = new byte[(int) fileLength];
        byte[] buf = new byte[BUFFER_SIZE];
        int bytesRead = fis.read(buf);

        int offset = 0;
        int chunk_count = 0;
        while (bytesRead != -1) {
            System.arraycopy(buf, 0, byteArray, offset, bytesRead);
            offset += bytesRead;// w  w w .  j  a v  a2s  .  c o  m
            bytesRead = fis.read(buf);
            chunk_count++;
        }

        BytesWritable value = new BytesWritable(byteArray);
        int len = (int) fileLength;
        value.setSize(len);
        filecount++;
        logger.info(this.getId() + ": " + filecount + ":" + key);
        writer.append(key, value);
        fis.close();
    } else {
        logger.warn("File " + file.getAbsolutePath() + " is too large to be "
                + "added to a sequence file (skipped).");
    }
}

From source file:eu.stratosphere.addons.parquet.SequenceFileSourceTest.java

License:Apache License

private void populateSeqFile(File f, String[] content) throws IOException {
    URI uri = f.toURI();/*from   w w w .j ava  2 s.  c o  m*/
    Configuration conf = new JobConf();
    FileSystem fs = FileSystem.get(uri, conf);
    Path path = new Path(uri);
    IntWritable key = new IntWritable();
    Text value = new Text();
    SequenceFile.Writer writer = null;
    try {
        writer = SequenceFile.createWriter(fs, conf, path, key.getClass(), value.getClass());
        for (int i = 0; i < content.length; i++) {
            key.set(i);
            value.set(content[i]);
            writer.append(key, value);
        }
    } finally {
        IOUtils.closeStream(writer);
    }
}

From source file:eu.stratosphere.myriad.driver.hadoop.MyriadRecordReader.java

License:Apache License

@Override
public boolean next(NullWritable key, Text value) throws IOException {
    final String v = this.socketReader.next();
    if (v == null) {
        return false;
    }/*from www. j a v a2  s. co  m*/
    value.set(v);
    return true;
}

From source file:full_MapReduce.FindBestAttributeMapper.java

License:Open Source License

private TextArrayWritable getValues(MapWritable value) {
    TextArrayWritable res = new TextArrayWritable();
    Text[] tmp_res = new Text[value.keySet().size()];

    int index = 0;
    for (Writable w1 : value.keySet()) {
        MapWritable mw = (MapWritable) value.get(w1);
        int nb_class = mw.size();
        Text prefered_class = new Text();
        IntWritable best_count = new IntWritable(Integer.MIN_VALUE);
        for (Writable w2 : mw.keySet()) {
            if (((IntWritable) mw.get(w2)).compareTo(best_count) > 0) {
                best_count = (IntWritable) mw.get(w2);
                prefered_class.set((Text) w2);
            }/*from  w  ww.j  av a  2s.  com*/
        }
        tmp_res[index++] = new Text(((Text) w1).toString() + " " + nb_class + " " + prefered_class.toString());
    }

    res.set(tmp_res);
    return res;
}