com.ailk.oci.ocnosql.tools.load.single.SingleColumnReducer.java Source code

Introduction

Here is the source code for com.ailk.oci.ocnosql.tools.load.single.SingleColumnReducer.java

Source

/**
 * Copyright 2010 The Apache Software Foundation
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.ailk.oci.ocnosql.tools.load.single;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.TreeSet;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.ReflectionUtils;

import com.ailk.oci.ocnosql.common.compress.Compress;
import com.ailk.oci.ocnosql.common.util.CommonConstants;
import com.ailk.oci.ocnosql.tools.load.single.SingleColumnImportTsv.TextArrayWritable;

/**
 * Emits sorted Puts.
 * Reads in all Puts from passed Iterator, sorts them, then emits
 * Puts in sorted order.  If lots of columns per row, it will use lots of
 * memory sorting.
 * @see HFileOutputFormat
 * @see KeyValueSortReducer
 */
public class SingleColumnReducer
        extends Reducer<ImmutableBytesWritable, TextArrayWritable, ImmutableBytesWritable, KeyValue> {
    String separator = ",";

    String record_separator = "`";

    private StringBuilder sb;

    private String curTime;

    TreeSet<KeyValue> map = null;
    byte[] family = null;

    private Compress compressor;

    protected void setup(Context context) throws IOException, InterruptedException {
        super.setup(context);

        sb = new StringBuilder();
        Date date = new Date();
        SimpleDateFormat formatter = new SimpleDateFormat("ddHHmm");
        curTime = formatter.format(date);

        //keyvalue?
        map = new TreeSet<KeyValue>(KeyValue.COMPARATOR);

        //
        record_separator = CommonConstants.RECORD_SEPARATOR;

        //???
        String familyStr = context.getConfiguration().get(CommonConstants.SINGLE_FAMILY);
        if (StringUtils.isEmpty(familyStr)) {
            String columns = context.getConfiguration().get(CommonConstants.COLUMNS);
            familyStr = columns.substring(0, columns.indexOf(":"));
        }
        family = Bytes.toBytes(familyStr);

        try {
            //
            initCompressInstance(context);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    }

    @Override
    protected void reduce(ImmutableBytesWritable row, Iterable<TextArrayWritable> texts, Context context)
            throws java.io.IOException, InterruptedException {
        //???
        String columName = curTime;

        map.clear();

        Put put = new Put(row.get());
        Iterator<TextArrayWritable> iter = texts.iterator();
        TextArrayWritable latterText = null;
        try {
            int i = 0;
            boolean isFirst = false; //?
            while (iter.hasNext()) {
                latterText = ((TextArrayWritable) iter.next()).clone();
                if (i == 0) {
                    isFirst = true;
                } else {
                    isFirst = false;
                }

                String dealStr = compressor.compress(latterText, isFirst, context.getConfiguration());
                if (!StringUtils.isEmpty(dealStr)) {
                    sb.append(dealStr);
                }
                sb = sb.append(record_separator);
                i++;
            }
            //?
            sb.deleteCharAt(sb.lastIndexOf(record_separator));
            put.add(family, columName.getBytes(), Bytes.toBytes(sb.toString()));
            sb.setLength(0);

            for (List<KeyValue> kvList : put.getFamilyMap().values()) {
                for (KeyValue kv : kvList) {
                    map.add(kv);
                }
            }

            for (KeyValue kv : map) {
                context.write(row, kv);
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    @SuppressWarnings("unchecked")
    private void initCompressInstance(Context context) throws ClassNotFoundException {
        //??
        String compressorName = context.getConfiguration().get(CommonConstants.COMPRESSOR);
        //???
        Class compressorClass = Class.forName(compressorName);
        compressor = (Compress) ReflectionUtils.newInstance(compressorClass, context.getConfiguration());
    }
}