Example usage for org.apache.hadoop.conf Configuration setInt

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration setInt.

Prototype

public void setInt(String name, int value) 

Source Link

Document

Set the value of the name property to an int.

Usage

From source file:com.moz.fiji.mapreduce.kvstore.impl.KeyValueStoreConfigSerializer.java

License:Apache License

/**
 * Given a set of KeyValueStores, add their configuration data to the provided Configuration.
 * This replaces any previous KeyValueStore configs in the Configuration.
 *
 * @param stores the map from names to KeyValueStores to serialize.
 * @param conf the Configuration to hold the stores.
 * @throws IOException if there is an error serializing the configuration of a store.
 *//*  w  w w.  j a va2  s  . co m*/
public void addStoreMapToConfiguration(Map<String, KeyValueStore<?, ?>> stores, Configuration conf)
        throws IOException {
    int i = 0;
    for (Map.Entry<String, KeyValueStore<?, ?>> entry : stores.entrySet()) {
        String storeName = entry.getKey();
        KeyValueStore<?, ?> store = entry.getValue();

        if (null != store) {
            KeyValueStoreConfiguration kvStoreConf = KeyValueStoreConfiguration.createInConfiguration(conf, i);

            // Set the default deserializer class to the same one that serialized it.
            // This is done first, so the store itself can override it.
            kvStoreConf.set(CONF_CLASS, store.getClass().getName());

            // Serialize the class' inner data itself.
            store.storeToConf(kvStoreConf);

            // Then store the name binding associated with this KeyValueStore.
            kvStoreConf.set(CONF_NAME, storeName);

            i++;
        }
    }

    // Record the number of KeyValueStore instances we wrote to the conf, so we can
    // deserialize the same number afterward.
    conf.setInt(CONF_KEY_VALUE_STORE_COUNT, i);
}

---

From source file:com.moz.fiji.mapreduce.kvstore.TestKeyValueStoreConfiguration.java

License:Apache License

@Test
public void testCopyFromConfiguration() {
    Configuration conf = new Configuration(false);
    conf.set("foo", "foo-value");
    conf.setInt("bar", 123);
    conf.setClass("qaz", String.class, Object.class);

    KeyValueStoreConfiguration kvStoreConf = KeyValueStoreConfiguration.fromConf(conf);
    assertEquals("foo-value", kvStoreConf.get("foo"));
    assertEquals(123, kvStoreConf.getInt("bar", 0));
    assertEquals(String.class, kvStoreConf.getClass("qaz", null));
}

---

From source file:com.moz.fiji.mapreduce.tools.FijiBulkLoad.java

License:Apache License

@Override
protected int run(List<String> nonFlagArgs) throws Exception {
    final Fiji fiji = Fiji.Factory.open(mTableURI, getConf());
    try {/*from  w  ww  . ja  v a2 s .c  om*/
        final FijiTable table = fiji.openTable(mTableURI.getTable());
        try {
            // Load the HFiles.
            //
            // TODO: Consolidate this logic in a single central place: We must consolidate the
            // logic to properly initialize a Configuration object to target a specific HBase
            // cluster (hence the manual override of the ZooKeeper quorum/client-port).
            //
            // The reason for this manual override here is : FijiBulkLoad needs a
            // Configuration to create an HBaseLoader for the HBase instance targeted at from
            // the table URI.  FijiTable does not expose its internal Configuration and
            // Fiji.getConf() is deprecated, so we have to construct one externally.
            final Configuration conf = getConf();

            conf.set(HConstants.ZOOKEEPER_QUORUM, Joiner.on(",").join(mTableURI.getZookeeperQuorumOrdered()));
            conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, mTableURI.getZookeeperClientPort());

            final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(2);
            Future<?> chmodTask = null;
            try {
                // We have to continually run chmod a+rw so that files newly created by splits will
                // be usable by hbase.  See https://issues.apache.org/jira/browse/HBASE-6422.
                // HBase proposes to solve this with the Secure Bulk Loader, but, for now, this
                // band-aid works.
                if (mChmodBackground) {
                    final Runnable chmodRunnable = new Runnable() {
                        static final int MAX_CONSECUTIVE_ERRORS = 5;

                        private int mNumConsecutiveErrors = 0;

                        /** {@inheritDoc} */
                        @Override
                        public void run() {
                            try {
                                recursiveGrantAllReadWritePermissions(mHFile);
                                mNumConsecutiveErrors = 0;
                            } catch (IOException ex) {
                                LOG.warn("recursiveGrantAllReadWritePermissions raised exception: {}", ex);
                                mNumConsecutiveErrors += 1;
                                if (mNumConsecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
                                    throw new RuntimeException("too many IOExceptions", ex);
                                }
                            }
                        }
                    };
                    chmodTask = executorService.scheduleAtFixedRate(chmodRunnable, 0, 1, TimeUnit.SECONDS);
                }
                // NOTE: HFileLoader never uses conf.
                final HFileLoader hFileLoader = HFileLoader.create(conf);
                // launch the load on a separate thread and wait to cancel if timeout is exceeded
                final Callable<Void> hFileLoadCallable = new Callable<Void>() {
                    public Void call() throws Exception {
                        hFileLoader.load(mHFile, table);
                        return null;
                    }
                };
                final Future<Void> hFileLoadTask = executorService.submit(hFileLoadCallable);
                try {
                    hFileLoadTask.get(mTimeoutSeconds, TimeUnit.SECONDS);
                } catch (TimeoutException ex) {
                    getPrintStream()
                            .println("Bulk-load failed due to a timeout after " + mTimeoutSeconds + "s.");
                    hFileLoadTask.cancel(true);
                    return FAILURE;
                } catch (ExecutionException executionException) {
                    // try to unpack the exception that bulk load raised
                    Exception cause = null;
                    try {
                        cause = (Exception) executionException.getCause();
                        if (cause == null) {
                            // There was no cause? Fall back to the original ExecutionException.
                            cause = executionException;
                        }
                    } catch (ClassCastException castException) {
                        // Cause wasn't an exception? Fall back to the original ExecutionException.
                        cause = executionException;
                    }
                    throw cause;
                }
            } finally {
                if (chmodTask != null) {
                    chmodTask.cancel(false);
                }
                executorService.shutdown();
            }
            return SUCCESS;
        } finally {
            ResourceUtils.releaseOrLog(table);
        }
    } finally {
        ResourceUtils.releaseOrLog(fiji);
    }
}

---

From source file:com.moz.fiji.schema.impl.hbase.HBaseFijiInstaller.java

License:Apache License

/** {@inheritDoc} */
@Override//from   ww  w . ja v  a  2  s  .  co m
public void install(final FijiURI uri, final HBaseFactory hbaseFactory, final Map<String, String> properties,
        final Configuration conf) throws IOException {
    if (uri.getInstance() == null) {
        throw new FijiInvalidNameException(
                String.format("Fiji URI '%s' does not specify a Fiji instance name", uri));
    }

    final HBaseAdminFactory adminFactory = hbaseFactory.getHBaseAdminFactory(uri);
    final HTableInterfaceFactory tableFactory = hbaseFactory.getHTableInterfaceFactory(uri);

    // TODO: Factor this in HBaseFiji
    conf.set(HConstants.ZOOKEEPER_QUORUM, Joiner.on(",").join(uri.getZookeeperQuorumOrdered()));
    conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, uri.getZookeeperClientPort());

    final HBaseAdmin hbaseAdmin = adminFactory.create(conf);
    try {
        if (hbaseAdmin
                .tableExists(FijiManagedHBaseTableName.getSystemTableName(uri.getInstance()).toString())) {
            throw new FijiAlreadyExistsException(String.format("Fiji instance '%s' already exists.", uri), uri);
        }
        LOG.info(String.format("Installing fiji instance '%s'.", uri));
        HBaseSystemTable.install(hbaseAdmin, uri, conf, properties, tableFactory);
        HBaseMetaTable.install(hbaseAdmin, uri);
        HBaseSchemaTable.install(hbaseAdmin, uri, conf, tableFactory);
        // Grant the current user all privileges on the instance just created, if security is enabled.
        final Fiji fiji = Fiji.Factory.open(uri, conf);
        try {
            if (fiji.isSecurityEnabled()) {
                FijiSecurityManager.Installer.installInstanceCreator(uri, conf, tableFactory);
            }
        } finally {
            fiji.release();
        }
    } finally {
        ResourceUtils.closeOrLog(hbaseAdmin);
    }
    LOG.info(String.format("Installed fiji instance '%s'.", uri));
}

---

From source file:com.moz.fiji.schema.impl.hbase.HBaseFijiInstaller.java

License:Apache License

/** {@inheritDoc} */
@Override// w w  w  .  j  av a2  s .  c o m
public void uninstall(final FijiURI uri, final HBaseFactory hbaseFactory, final Configuration conf)
        throws IOException {
    if (uri.getInstance() == null) {
        throw new FijiInvalidNameException(
                String.format("Fiji URI '%s' does not specify a Fiji instance name", uri));
    }
    final HBaseAdminFactory adminFactory = hbaseFactory.getHBaseAdminFactory(uri);
    final HTableInterfaceFactory htableFactory = hbaseFactory.getHTableInterfaceFactory(uri);

    // TODO: Factor this in HBaseFiji
    conf.set(HConstants.ZOOKEEPER_QUORUM, Joiner.on(",").join(uri.getZookeeperQuorumOrdered()));
    conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, uri.getZookeeperClientPort());

    LOG.info(String.format("Removing the fiji instance '%s'.", uri.getInstance()));

    final ProtocolVersion systemVersion = getSystemVersion(uri, conf, htableFactory);
    if (systemVersion.compareTo(Versions.SYSTEM_2_0) < 0) {
        uninstallSystem_1_0(uri, conf, adminFactory);
    } else if (systemVersion.compareTo(Versions.SYSTEM_2_0) == 0) {
        uninstallSystem_2_0(uri, conf, adminFactory);
    } else {
        throw new InternalFijiError(String.format("Unknown System version %s.", systemVersion));
    }

    LOG.info("Removed fiji instance '{}'.", uri.getInstance());
}

---

From source file:com.moz.fiji.schema.tools.LsTool.java

License:Apache License

/**
 * Returns a set of instance names./*from   w  ww. j  av  a2  s . c om*/
 *
 * @param hbaseURI URI of the HBase instance to list the content of.
 * @return ordered set of instance names.
 * @throws IOException on I/O error.
 */
protected static Set<String> getInstanceNames(FijiURI hbaseURI) throws IOException {
    // TODO(SCHEMA-188): Consolidate this logic in a single central place:
    final Configuration conf = HBaseConfiguration.create();
    conf.set(HConstants.ZOOKEEPER_QUORUM, Joiner.on(",").join(hbaseURI.getZookeeperQuorumOrdered()));
    conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, hbaseURI.getZookeeperClientPort());
    final HBaseAdmin hbaseAdmin = HBaseFactory.Provider.get().getHBaseAdminFactory(hbaseURI).create(conf);

    try {
        final Set<String> instanceNames = Sets.newTreeSet();
        for (HTableDescriptor hTableDescriptor : hbaseAdmin.listTables()) {
            final String instanceName = parseInstanceName(hTableDescriptor.getNameAsString());
            if (null != instanceName) {
                instanceNames.add(instanceName);
            }
        }
        return instanceNames;
    } finally {
        ResourceUtils.closeOrLog(hbaseAdmin);
    }
}

---

From source file:com.navercorp.pinpoint.common.hbase.HBaseAsyncOperationFactory.java

License:Apache License

public static HBaseAsyncOperation create(Configuration configuration) throws IOException {
    boolean enableAsyncMethod = configuration.getBoolean(ENABLE_ASYNC_METHOD, DEFAULT_ENABLE_ASYNC_METHOD);
    if (!enableAsyncMethod) {
        return DisabledHBaseAsyncOperation.INSTANCE;
    }//from   w w  w . jav a2  s.  c  om

    int queueSize = configuration.getInt(ASYNC_IN_QUEUE_SIZE, DEFAULT_ASYNC_IN_QUEUE_SIZE);

    if (configuration.get(ASYNC_PERIODIC_FLUSH_TIME, null) == null) {
        configuration.setInt(ASYNC_PERIODIC_FLUSH_TIME, DEFAULT_ASYNC_PERIODIC_FLUSH_TIME);
    }

    if (configuration.get(ASYNC_RETRY_COUNT, null) == null) {
        configuration.setInt(ASYNC_RETRY_COUNT, DEFAULT_ASYNC_RETRY_COUNT);
    }

    return new HBaseAsyncTemplate(configuration, queueSize);
}

---

From source file:com.nec.strudel.tkvs.store.omid.OmidStore.java

License:Apache License

@Override
public OmidDbServer createDbServer(String dbName, Properties props) {
    /**//ww  w. j av a2  s .  c o m
     * NOTE without this, HBaseConfiguration fails to find
     * hbase-default.xml
     */
    Thread.currentThread().setContextClassLoader(HBaseConfiguration.class.getClassLoader());
    Configuration conf = HBaseConfiguration.create();
    conf.set("tso.host", "localhost");
    conf.setInt("tso.port", TSOPORT);
    conf.set("hbase.zookeeper.quorum", "localhost");
    conf.setInt("hbase.zookeeper.property.clientPort", CLIENTPORT);
    //Max Transactions that omid server will handle at the same time,
    //This won't be in effect here, to set this parameter.
    //modify the value in omid.sh
    //conf.setInt("omid.maxItems", 1);
    String[] tsoHosts = null;
    for (Map.Entry<Object, Object> e : props.entrySet()) {
        conf.set(e.getKey().toString(), e.getValue().toString());
        if (e.getKey().toString().equals("tso.host")) {
            tsoHosts = e.getValue().toString().split(",");
        }
    }
    try {
        return new OmidDbServer(dbName, conf, tableDescriptor(dbName), tsoHosts, new BackoffPolicy(props));
    } catch (ZooKeeperConnectionException e) {
        throw new TkvStoreException("OmidStore failed to create (ZooKeeper connection)", e);
    } catch (IOException e) {
        throw new TkvStoreException("OmidStore failed to create (IOException)", e);
    } catch (OmidInstantiationException e) {
        throw new TkvStoreException("OmidStore failed to create (Omid instanitation)", e);
    }
}

---

From source file:com.ning.metrics.action.binder.modules.FileSystemAccessProvider.java

License:Apache License

@Inject
public FileSystemAccessProvider(final ActionCoreConfig actionCoreConfig) throws IOException {
    final Configuration hadoopConfig = new Configuration();

    final String hfsHost = actionCoreConfig.getNamenodeUrl();
    if (hfsHost.isEmpty()) {
        // Local filesystem, for testing
        hadoopConfig.set("fs.default.name", "file:///");
    } else {// ww  w . j  a va  2  s .  c om
        hadoopConfig.set("fs.default.name", hfsHost);
    }

    // Bump the default timeout
    hadoopConfig.setInt("dfs.socket.timeout", actionCoreConfig.getHadoopSocketTimeOut());
    hadoopConfig.setBoolean("fs.automatic.close", false);
    hadoopConfig.setLong("dfs.block.size", actionCoreConfig.getHadoopBlockSize());
    hadoopConfig.set("hadoop.job.ugi", actionCoreConfig.getHadoopUgi());
    hadoopConfig.setStrings("io.serializations", HadoopThriftWritableSerialization.class.getName(),
            HadoopThriftEnvelopeSerialization.class.getName(),
            HadoopSmileOutputStreamSerialization.class.getName(),
            "org.apache.hadoop.io.serializer.WritableSerialization", actionCoreConfig.getSerializations());

    fileSystemAccess = new FileSystemAccess(hadoopConfig);
}

---

From source file:com.nistfortunetellers.cleaning.NISTClean.java

License:Apache License

static void addLaneIDsToConfig(Configuration config) {
    BufferedReader br = null;/*w  w  w . ja v  a  2 s  .  com*/
    String line = "";
    String splitChar = ",";
    try {

        br = new BufferedReader(new FileReader(LANE_ZONE_MAPPINGS));
        //Skip First Line, since it is the keys.
        br.readLine();
        //Read In All Lines In File
        while ((line = br.readLine()) != null) {
            //Split by Comma
            String[] splits = line.split(splitChar);

            /* Parse Relevant Docs */
            //will be used as key
            String laneID = "";
            //associated val for key
            int zoneID = 0;
            //make sure we aren't just on a blank line
            if (splits.length >= 2) {
                laneID = splits[0].trim();
                try {
                    zoneID = Integer.parseInt(splits[1]);
                } catch (NumberFormatException e) {
                    //If format doesn't work out so well, no worries. Just skip.
                    //System.out.println("--Skipped line Line: " + line.replace('\n', ' '));
                    continue;
                }
            } else {
                //probably means we're done, but continue just in case.
                continue;
            }
            //DEBUG: Print Output
            System.out.println("LaneID: " + laneID + ", zoneID: " + zoneID);
            //Now actually set the config!
            config.setInt(laneID, zoneID);
        }

    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (br != null) {
            try {
                br.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• »