Example usage for org.apache.hadoop.conf Configuration getLong

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration getLong.

Prototype

public long getLong(String name, long defaultValue) 

Source Link

Document

Get the value of the name property as a long.

Usage

From source file:org.apache.phoenix.mapreduce.MultiHfileOutputFormat.java

License:Apache License

/**
 * //from ww  w.ja v  a2  s .  c o m
 * @param context
 * @return
 * @throws IOException 
 */
static <V extends Cell> RecordWriter<TableRowkeyPair, V> createRecordWriter(final TaskAttemptContext context)
        throws IOException {
    // Get the path of the temporary output file
    final Path outputPath = FileOutputFormat.getOutputPath(context);
    final Path outputdir = new FileOutputCommitter(outputPath, context).getWorkPath();
    final Configuration conf = context.getConfiguration();
    final FileSystem fs = outputdir.getFileSystem(conf);

    final long maxsize = conf.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE);
    // Invented config.  Add to hbase-*.xml if other than default compression.
    final String defaultCompressionStr = conf.get("hfile.compression", Compression.Algorithm.NONE.getName());
    final Algorithm defaultCompression = AbstractHFileWriter.compressionByName(defaultCompressionStr);
    final boolean compactionExclude = conf.getBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
            false);

    return new RecordWriter<TableRowkeyPair, V>() {
        // Map of families to writers and how much has been output on the writer.
        private final Map<byte[], WriterLength> writers = new TreeMap<byte[], WriterLength>(
                Bytes.BYTES_COMPARATOR);
        private byte[] previousRow = HConstants.EMPTY_BYTE_ARRAY;
        private final byte[] now = Bytes.toBytes(EnvironmentEdgeManager.currentTimeMillis());
        private boolean rollRequested = false;

        @Override
        public void write(TableRowkeyPair row, V cell) throws IOException {
            KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
            // null input == user explicitly wants to flush
            if (row == null && kv == null) {
                rollWriters();
                return;
            }

            // phoenix-2216: start : extract table name from the rowkey
            String tableName = row.getTableName();
            byte[] rowKey = row.getRowkey().get();
            long length = kv.getLength();
            byte[] family = CellUtil.cloneFamily(kv);
            byte[] tableAndFamily = join(tableName, Bytes.toString(family));
            WriterLength wl = this.writers.get(tableAndFamily);
            // phoenix-2216: end

            // If this is a new column family, verify that the directory exists
            if (wl == null) {
                // phoenix-2216: start : create a directory for table and family within the output dir 
                Path tableOutputPath = CsvBulkImportUtil.getOutputPath(outputdir, tableName);
                fs.mkdirs(new Path(tableOutputPath, Bytes.toString(family)));
                // phoenix-2216: end
            }

            // If any of the HFiles for the column families has reached
            // maxsize, we need to roll all the writers
            if (wl != null && wl.written + length >= maxsize) {
                this.rollRequested = true;
            }

            // This can only happen once a row is finished though
            if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) {
                rollWriters();
            }

            // create a new WAL writer, if necessary
            if (wl == null || wl.writer == null) {
                // phoenix-2216: start : passed even the table name
                wl = getNewWriter(tableName, family, conf);
                // phoenix-2216: end
            }

            // we now have the proper WAL writer. full steam ahead
            kv.updateLatestStamp(this.now);
            wl.writer.append(kv);
            wl.written += length;

            // Copy the row so we know when a row transition.
            this.previousRow = rowKey;
        }

        private void rollWriters() throws IOException {
            for (WriterLength wl : this.writers.values()) {
                if (wl.writer != null) {
                    LOG.info("Writer=" + wl.writer.getPath()
                            + ((wl.written == 0) ? "" : ", wrote=" + wl.written));
                    close(wl.writer);
                }
                wl.writer = null;
                wl.written = 0;
            }
            this.rollRequested = false;
        }

        /* Create a new StoreFile.Writer.
         * @param family
         * @return A WriterLength, containing a new StoreFile.Writer.
         * @throws IOException
         */
        @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "BX_UNBOXING_IMMEDIATELY_REBOXED", justification = "Not important")
        private WriterLength getNewWriter(final String tableName, byte[] family, Configuration conf)
                throws IOException {

            WriterLength wl = new WriterLength();
            Path tableOutputPath = CsvBulkImportUtil.getOutputPath(outputdir, tableName);
            Path familydir = new Path(tableOutputPath, Bytes.toString(family));

            // phoenix-2216: start : fetching the configuration properties that were set to the table.
            // create a map from column family to the compression algorithm for the table.
            final Map<byte[], Algorithm> compressionMap = createFamilyCompressionMap(conf, tableName);
            final Map<byte[], BloomType> bloomTypeMap = createFamilyBloomTypeMap(conf, tableName);
            final Map<byte[], Integer> blockSizeMap = createFamilyBlockSizeMap(conf, tableName);
            // phoenix-2216: end

            String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY);
            final Map<byte[], DataBlockEncoding> datablockEncodingMap = createFamilyDataBlockEncodingMap(conf,
                    tableName);
            final DataBlockEncoding overriddenEncoding;
            if (dataBlockEncodingStr != null) {
                overriddenEncoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
            } else {
                overriddenEncoding = null;
            }

            Algorithm compression = compressionMap.get(family);
            compression = compression == null ? defaultCompression : compression;
            BloomType bloomType = bloomTypeMap.get(family);
            bloomType = bloomType == null ? BloomType.NONE : bloomType;
            Integer blockSize = blockSizeMap.get(family);
            blockSize = blockSize == null ? HConstants.DEFAULT_BLOCKSIZE : blockSize;
            DataBlockEncoding encoding = overriddenEncoding;
            encoding = encoding == null ? datablockEncodingMap.get(family) : encoding;
            encoding = encoding == null ? DataBlockEncoding.NONE : encoding;
            Configuration tempConf = new Configuration(conf);
            tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f);
            HFileContextBuilder contextBuilder = new HFileContextBuilder().withCompression(compression)
                    .withChecksumType(HStore.getChecksumType(conf))
                    .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blockSize);
            contextBuilder.withDataBlockEncoding(encoding);
            HFileContext hFileContext = contextBuilder.build();

            wl.writer = new StoreFile.WriterBuilder(conf, new CacheConfig(tempConf), fs)
                    .withOutputDir(familydir).withBloomType(bloomType).withComparator(KeyValue.COMPARATOR)
                    .withFileContext(hFileContext).build();

            // join and put it in the writers map .
            // phoenix-2216: start : holds a map of writers where the 
            //                       key in the map is a join byte array of table name and family.
            byte[] tableAndFamily = join(tableName, Bytes.toString(family));
            this.writers.put(tableAndFamily, wl);
            // phoenix-2216: end
            return wl;
        }

        private void close(final StoreFile.Writer w) throws IOException {
            if (w != null) {
                w.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
                        Bytes.toBytes(EnvironmentEdgeManager.currentTimeMillis()));
                w.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY,
                        Bytes.toBytes(context.getTaskAttemptID().toString()));
                w.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, Bytes.toBytes(true));
                w.appendFileInfo(StoreFile.EXCLUDE_FROM_MINOR_COMPACTION_KEY, Bytes.toBytes(compactionExclude));
                w.appendTrackedTimestampsToMetadata();
                w.close();
            }
        }

        @Override
        public void close(TaskAttemptContext c) throws IOException, InterruptedException {
            for (WriterLength wl : this.writers.values()) {
                close(wl.writer);
            }
        }
    };
}

---

From source file:org.apache.phoenix.mapreduce.OrphanViewTool.java

License:Apache License

/**
 * Examples for input arguments://from  ww  w. j  a v a  2s.co m
 * -c : cleans orphan views
 * -c -op /tmp/ : cleans orphan views and links, and logs their names to the files named Orphan*.txt in /tmp/
 * -i : identifies orphan views and links, and prints their names on the console
 * -i -op /tmp/ : identifies orphan views and links, and logs the name of their names to files named Orphan*.txt in /tmp/
 * -c -ip /tmp/ : cleans the views listed in files at /tmp/
 */
@Override
public int run(String[] args) throws Exception {
    Connection connection = null;
    try {
        final Configuration configuration = HBaseConfiguration.addHbaseResources(getConf());

        try {
            parseOptions(args);
        } catch (IllegalStateException e) {
            printHelpAndExit(e.getMessage(), getOptions());
        }
        if (outputPath != null) {
            // Create files to log orphan views and links
            for (int i = VIEW; i < ORPHAN_TYPE_COUNT; i++) {
                File file = new File(outputPath + fileName[i]);
                if (file.exists()) {
                    file.delete();
                }
                file.createNewFile();
                writer[i] = new BufferedWriter(new FileWriter(file));
            }
        }
        Properties props = new Properties();
        long scn = System.currentTimeMillis() - ageMs;
        props.setProperty("CurrentSCN", Long.toString(scn));
        connection = ConnectionUtil.getInputConnection(configuration, props);
        PhoenixConnection phoenixConnection = connection.unwrap(PhoenixConnection.class);
        identifyOrphanViews(phoenixConnection);
        if (clean) {
            // Close the connection with SCN
            phoenixConnection.close();
            connection = ConnectionUtil.getInputConnection(configuration);
            phoenixConnection = connection.unwrap(PhoenixConnection.class);
            // Take a snapshot of system tables to be modified
            createSnapshot(phoenixConnection, scn);
        }
        for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) {
            try {
                dropOrLogOrphanViews(phoenixConnection, configuration, entry.getKey());
            } catch (Exception e) {
                // Ignore
            }
        }
        ;
        if (clean) {
            // Wait for the view drop tasks in the SYSTEM.TASK table to be processed
            long timeInterval = configuration.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
                    QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS);
            Thread.sleep(maxViewLevel * timeInterval);
            // Clean up any remaining orphan view records from system tables
            for (Map.Entry<Key, View> entry : orphanViewSet.entrySet()) {
                try {
                    forcefullyDropView(phoenixConnection, entry.getKey());
                } catch (Exception e) {
                    // Ignore
                }
            }
            ;
        }
        if (inputPath == null) {
            removeOrLogOrphanLinks(phoenixConnection);
        } else {
            readAndRemoveOrphanLinks(phoenixConnection);
        }
        return 0;
    } catch (Exception ex) {
        LOG.error("Orphan View Tool : An exception occurred " + ExceptionUtils.getMessage(ex) + " at:\n"
                + ExceptionUtils.getStackTrace(ex));
        return -1;
    } finally {
        closeConnectionAndFiles(connection);
    }
}

---

From source file:org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.java

License:Apache License

public static long getBatchSize(final Configuration configuration) throws SQLException {
    Preconditions.checkNotNull(configuration);
    long batchSize = configuration.getLong(UPSERT_BATCH_SIZE, DEFAULT_UPSERT_BATCH_SIZE);
    if (batchSize <= 0) {
        Connection conn = ConnectionUtil.getOutputConnection(configuration);
        batchSize = ((PhoenixConnection) conn).getMutateBatchSize();
        conn.close();/*  w ww .ja  va 2s .com*/
    }
    configuration.setLong(UPSERT_BATCH_SIZE, batchSize);
    return batchSize;
}

---

From source file:org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.java

License:Apache License

public static long getScrutinyBatchSize(Configuration configuration) {
    Preconditions.checkNotNull(configuration);
    return configuration.getLong(SCRUTINY_BATCH_SIZE, DEFAULT_SCRUTINY_BATCH_SIZE);
}

---

From source file:org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.java

License:Apache License

public static long getScrutinyExecuteTimestamp(Configuration configuration) {
    Preconditions.checkNotNull(configuration);
    long ts = configuration.getLong(SCRUTINY_EXECUTE_TIMESTAMP, -1);
    Preconditions.checkArgument(ts != -1);
    return ts;/*from   www . j a v  a  2s . c o  m*/
}

---

From source file:org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.java

License:Apache License

public static long getScrutinyOutputMax(Configuration configuration) {
    Preconditions.checkNotNull(configuration);
    long maxRows = configuration.getLong(SCRUTINY_OUTPUT_MAX, -1);
    Preconditions.checkArgument(maxRows != -1);
    return maxRows;
}

---

From source file:org.apache.phoenix.query.GuidePostsCache.java

License:Apache License

public GuidePostsCache(ConnectionQueryServices queryServices, Configuration config) {
    this.queryServices = Objects.requireNonNull(queryServices);
    // Number of millis to expire cache values after write
    final long statsUpdateFrequency = config.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
            QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
    // Maximum number of entries (tables) to store in the cache at one time
    final long maxTableStatsCacheSize = config.getLong(QueryServices.STATS_MAX_CACHE_SIZE,
            QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
    cache = CacheBuilder.newBuilder()/* w  ww  .  j ava  2  s .c om*/
            // Expire entries a given amount of time after they were written
            .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
            // Maximum total weight (size in bytes) of stats entries
            .maximumWeight(maxTableStatsCacheSize)
            // Defer actual size to the PTableStats.getEstimatedSize()
            .weigher(new Weigher<GuidePostsKey, GuidePostsInfo>() {
                @Override
                public int weigh(GuidePostsKey key, GuidePostsInfo info) {
                    return info.getEstimatedSize();
                }
            })
            // Log removals at TRACE for debugging
            .removalListener(new PhoenixStatsCacheRemovalListener())
            // Automatically load the cache when entries are missing
            .build(new StatsLoader());
}

---

From source file:org.apache.phoenix.query.TableStatsCache.java

License:Apache License

public TableStatsCache(ConnectionQueryServices queryServices, Configuration config) {
    this.queryServices = Objects.requireNonNull(queryServices);
    // Number of millis to expire cache values after write
    final long statsUpdateFrequency = config.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
            QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS);
    // Maximum number of entries (tables) to store in the cache at one time
    final long maxTableStatsCacheSize = config.getLong(QueryServices.STATS_MAX_CACHE_SIZE,
            QueryServicesOptions.DEFAULT_STATS_MAX_CACHE_SIZE);
    cache = CacheBuilder.newBuilder()// www  .j a  va2s  . c  o  m
            // Expire entries a given amount of time after they were written
            .expireAfterWrite(statsUpdateFrequency, TimeUnit.MILLISECONDS)
            // Maximum total weight (size in bytes) of stats entries
            .maximumWeight(maxTableStatsCacheSize)
            // Defer actual size to the PTableStats.getEstimatedSize()
            .weigher(new Weigher<ImmutableBytesPtr, PTableStats>() {
                @Override
                public int weigh(ImmutableBytesPtr key, PTableStats stats) {
                    return stats.getEstimatedSize();
                }
            })
            // Log removals at TRACE for debugging
            .removalListener(new PhoenixStatsCacheRemovalListener())
            // Automatically load the cache when entries are missing
            .build(new StatsLoader());
}

---

From source file:org.apache.phoenix.schema.stats.DefaultStatisticsCollector.java

License:Apache License

private void initGuidepostDepth() throws IOException {
    // First check is if guidepost info set on statement itself
    if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) {
        int guidepostPerRegion = 0;
        long guidepostWidth = QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES;
        if (guidePostPerRegionBytes != null) {
            guidepostPerRegion = PInteger.INSTANCE.getCodec().decodeInt(guidePostPerRegionBytes, 0,
                    SortOrder.getDefault());
        }//from  www .  j  a  va 2s.c  om
        if (guidePostWidthBytes != null) {
            guidepostWidth = PLong.INSTANCE.getCodec().decodeInt(guidePostWidthBytes, 0,
                    SortOrder.getDefault());
        }
        this.guidePostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
                env.getRegion().getTableDesc());
    } else {
        long guidepostWidth = -1;
        HTableInterface htable = null;
        try {
            // Next check for GUIDE_POST_WIDTH on table
            htable = env.getTable(SchemaUtil.getPhysicalTableName(
                    PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, env.getConfiguration()));
            Get get = new Get(ptableKey);
            get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
                    PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
            Result result = htable.get(get);
            if (!result.isEmpty()) {
                Cell cell = result.listCells().get(0);
                guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(),
                        cell.getValueOffset(), SortOrder.getDefault());
            }
        } finally {
            if (htable != null) {
                try {
                    htable.close();
                } catch (IOException e) {
                    LOG.warn("Failed to close " + htable.getName(), e);
                }
            }
        }
        if (guidepostWidth >= 0) {
            this.guidePostDepth = guidepostWidth;
        } else {
            // Last use global config value
            Configuration config = env.getConfiguration();
            this.guidePostDepth = StatisticsUtil.getGuidePostDepth(
                    config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
                            QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION),
                    config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
                            QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES),
                    env.getRegion().getTableDesc());
        }
    }
}

---

From source file:org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator.java

License:Apache License

/**
 * Determines the number of reducers to be used.
 *
 * @param job job instance//from   ww  w .  ja  v a2 s .  co m
 * @param mapReduceOper
 * @throws java.io.IOException
 */
@Override
public int estimateNumberOfReducers(Job job, MapReduceOper mapReduceOper) throws IOException {
    Configuration conf = job.getConfiguration();

    long bytesPerReducer = conf.getLong(BYTES_PER_REDUCER_PARAM, DEFAULT_BYTES_PER_REDUCER);
    int maxReducers = conf.getInt(MAX_REDUCER_COUNT_PARAM, DEFAULT_MAX_REDUCER_COUNT_PARAM);

    List<POLoad> poLoads = PlanHelper.getPhysicalOperators(mapReduceOper.mapPlan, POLoad.class);
    long totalInputFileSize = getTotalInputFileSize(conf, poLoads, job);

    log.info("BytesPerReducer=" + bytesPerReducer + " maxReducers=" + maxReducers + " totalInputFileSize="
            + totalInputFileSize);

    // if totalInputFileSize == -1, we couldn't get the input size so we can't estimate.
    if (totalInputFileSize == -1) {
        return -1;
    }

    int reducers = (int) Math.ceil((double) totalInputFileSize / bytesPerReducer);
    reducers = Math.max(1, reducers);
    reducers = Math.min(maxReducers, reducers);

    return reducers;
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• »