Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name, String defaultValue) 

Source Link

Document

Get the value of the name.

Usage

From source file:com.alexholmes.hadooputils.sort.LzoDelimitedLineRecordReader.java

License:Apache License

@Override
protected void initialize(Configuration job, FileSplit split) throws IOException {
    start = split.getStart();//from   w w  w . j  a va2  s . com
    end = start + split.getLength();
    final Path file = split.getPath();

    FileSystem fs = file.getFileSystem(job);
    CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(job);
    final CompressionCodec codec = compressionCodecs.getCodec(file);
    if (codec == null) {
        throw new IOException("No codec for file " + file + " not found, cannot run");
    }

    // open the file and seek to the start of the split
    fileIn = fs.open(split.getPath());

    // creates input stream and also reads the file header
    String rowDelim = job.get("textinputformat.record.delimiter", null);
    if (rowDelim != null) {
        byte[] hexcode = SortConfig.getHexDelimiter(rowDelim);
        in = new DelimitedLineReader(fileIn, job, (hexcode != null) ? hexcode : rowDelim.getBytes());
    } else {
        in = new DelimitedLineReader(codec.createInputStream(fileIn), job);
    }

    if (start != 0) {
        fileIn.seek(start);

        // read and ignore the first line
        in.readLine(new Text());
        start = fileIn.getPos();
    }

    this.pos = start;
}

---

From source file:com.alibaba.wasp.fserver.EntityGroupSplitPolicy.java

License:Apache License

static Class<? extends EntityGroupSplitPolicy> getSplitPolicyClass(FTable ftd, Configuration conf)
        throws IOException {
    String className = conf.get(FConstants.WASP_ENTITYGROUP_SPLIT_POLICY_KEY,
            DEFAULT_SPLIT_POLICY_CLASS.getName());

    try {/*from  w  w  w . j  a v  a  2  s.  c om*/
        Class<? extends EntityGroupSplitPolicy> clazz = Class.forName(className)
                .asSubclass(EntityGroupSplitPolicy.class);
        return clazz;
    } catch (Exception e) {
        throw new IOException("Unable to load configured entityGroup split policy '" + className
                + "' for table '" + ftd.getTableName() + "'", e);
    }
}

---

From source file:com.alibaba.wasp.fserver.FServer.java

License:Apache License

/**
 * Starts a FServer at the default location
 * //w  w  w .  jav  a2s .co  m
 * @param conf
 * @throws java.io.IOException
 * @throws InterruptedException
 */
public FServer(Configuration conf) throws IOException, InterruptedException {
    this.conf = conf;
    this.isOnline = false;
    // Set how many times to retry talking to another server over FConnection.
    FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG);

    // Config'ed params
    this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000);

    this.sleeper = new Sleeper(this.msgInterval, this);

    this.numEntityGroupsToReport = conf.getInt("wasp.fserver.numentitygroupstoreport", 10);

    this.rpcTimeout = conf.getInt(FConstants.WASP_RPC_TIMEOUT_KEY, FConstants.DEFAULT_WASP_RPC_TIMEOUT);

    this.abortRequested = false;
    this.stopped = false;
    this.actionManager = new StorageActionManager(conf);

    // Server to handle client requests.
    String hostname = Strings
            .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.fserver.dns.interface", "default"),
                    conf.get("wasp.fserver.dns.nameserver", "default")));
    int port = conf.getInt(FConstants.FSERVER_PORT, FConstants.DEFAULT_FSERVER_PORT);
    // Creation of a HSA will force a resolve.
    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }

    this.rpcServer = WaspRPC.getServer(FServer.class, this,
            new Class<?>[] { ClientProtocol.class, AdminProtocol.class, WaspRPCErrorHandler.class,
                    OnlineEntityGroups.class },
            initialIsa.getHostName(), // BindAddress is
            // IP we got for
            // this server.
            initialIsa.getPort(), conf);
    // Set our address.
    this.isa = this.rpcServer.getListenerAddress();

    this.leases = new Leases(conf.getInt(FConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));

    this.startcode = System.currentTimeMillis();

    int maxThreads = conf.getInt("wasp.transaction.threads.max", 150);

    this.pool = new ThreadPoolExecutor(1, maxThreads, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
            new DaemonThreadFactory("thread factory"));
    ((ThreadPoolExecutor) this.pool).allowCoreThreadTimeOut(true);

    this.scannerLeaseTimeoutPeriod = conf.getInt(FConstants.WASP_CLIENT_SCANNER_TIMEOUT_PERIOD,
            FConstants.DEFAULT_WASP_CLIENT_SCANNER_TIMEOUT_PERIOD);

    this.driver = new BaseDriver(this);
    this.splitThread = new SplitThread(this);
    this.globalEntityGroup = new GlobalEntityGroup(this);
}

---

From source file:com.alibaba.wasp.master.FMaster.java

License:Apache License

/**
 * Initializes the FMaster. The steps are as follows:
 * <p>/* w  ww . j ava 2 s.c  om*/
 * <ol>
 * <li>Initialize FMaster RPC and address
 * <li>Connect to ZooKeeper.
 * </ol>
 * <p>
 * Remaining steps of initialization occur in {@link #run()} so that they run
 * in their own thread rather than within the context of the constructor.
 * 
 * @throws InterruptedException
 */
public FMaster(final Configuration conf) throws IOException, KeeperException, InterruptedException {
    this.conf = new Configuration(conf);
    // Set how many times to retry talking to another server over HConnection.
    FConnectionManager.setServerSideFConnectionRetries(this.conf, LOG);
    // Server to handle client requests.
    String hostname = Strings
            .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.master.dns.interface", "default"),
                    conf.get("wasp.master.dns.nameserver", "default")));
    int port = conf.getInt(FConstants.MASTER_PORT, FConstants.DEFAULT_MASTER_PORT);
    // Creation of a ISA will force a resolve.
    InetSocketAddress initialIsa = new InetSocketAddress(hostname, port);
    if (initialIsa.getAddress() == null) {
        throw new IllegalArgumentException("Failed resolve of " + initialIsa);
    }
    this.rpcServer = WaspRPC.getServer(FMaster.class, this,
            new Class<?>[] { FMasterMonitorProtocol.class, FMasterAdminProtocol.class,
                    FServerStatusProtocol.class, FMetaServerProtocol.class },
            initialIsa.getHostName(), // BindAddress is IP we got for this server.
            initialIsa.getPort(), conf);
    // Set our address.
    this.isa = this.rpcServer.getListenerAddress();
    this.serverName = new ServerName(this.isa.getHostName(), this.isa.getPort(), System.currentTimeMillis());

    // set the thread name now we have an address
    setName(MASTER + "-" + this.serverName.toString());

    this.zooKeeper = new ZooKeeperWatcher(conf, MASTER + ":" + isa.getPort(), this, true);

    // metrics interval: using the same property as fserver.
    this.msgInterval = conf.getInt("wasp.fserver.msginterval", 3 * 1000);

    this.metricsMaster = new MetricsMaster(new MetricsMasterWrapperImpl(this));
}

---

From source file:com.alibaba.wasp.meta.FMetaScanner.java

License:Apache License

public static void metaScan(Configuration configuration, MetaScannerVisitor visitor, byte[] tableName,
        byte[] row, int rowLimit) throws IOException {
    int rowUpperLimit = rowLimit > 0 ? rowLimit : Integer.MAX_VALUE;

    // if row is not null, we want to use the startKey of the row's entityGroup
    // as//from  w w  w. ja v a2  s .c  om
    // the startRow for the meta scan.
    byte[] startRow = tableName;
    if (tableName == null || tableName.length == 0) {
        // Full META scan
        startRow = FConstants.EMPTY_START_ROW;
    }
    String metaTableString = configuration.get(FConstants.METASTORE_TABLE, FConstants.DEFAULT_METASTORE_TABLE);
    HTable fmetaTable = new HTable(configuration, metaTableString);
    // Scan over each meta entityGroup
    int rows = Math.min(rowLimit, configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
            HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));

    final Scan scan = new Scan(startRow);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scanning " + metaTableString + " starting at row=" + Bytes.toStringBinary(startRow)
                + " for max=" + rowUpperLimit + " rows ");
    }
    scan.addFamily(FConstants.CATALOG_FAMILY);
    scan.setCaching(rows);
    if (tableName == null || tableName.length == 0) {
        FilterList allFilters = new FilterList();
        allFilters.addFilter(new PrefixFilter(tableName));
        scan.setFilter(allFilters);
    }
    int processedRows = 0;
    try {
        ResultScanner scanner = fmetaTable.getScanner(scan);
        for (Result r = scanner.next(); r != null; r = scanner.next()) {
            if (processedRows >= rowUpperLimit) {
                break;
            }
            if (Bytes.startsWith(r.getRow(), FConstants.TABLEROW_PREFIX)) {
                continue;
            }
            if (!visitor.processRow(r))
                break; // exit completely
            processedRows++;
        }
        // here, we didn't break anywhere. Check if we have more rows
    } finally {
        // Close scanner
        fmetaTable.close();
    }
}

---

From source file:com.alibaba.wasp.meta.FMetaServicesImplWithoutRetry.java

License:Apache License

/**
 * Use HTable to connect to the meta table, will use the HTable to
 * create\drop\alter\get Table and so on.
 *///w  w w.ja v a  2  s  .  c om
@Override
public boolean connect() {
    try {
        Configuration conf = getConf();
        metaTable = conf.get(FConstants.METASTORE_TABLE, FConstants.DEFAULT_METASTORE_TABLE);
        hbaseActionManager = new StorageActionManager(conf);
    } catch (Exception e) {
        return false;
    }
    return true;
}

---

From source file:com.alibaba.wasp.meta.FMetaUtil.java

License:Apache License

/**
 * Use HBaseAdmin to check the meta table exists. if not exists create it. if
 * exists just return./*from  www.  j av a2 s.  c  o  m*/
 */
public static void checkAndInit(Configuration conf) throws MetaException {
    try {
        HBaseAdmin admin = new HBaseAdmin(conf);
        String metaTable = conf.get(FConstants.METASTORE_TABLE, FConstants.DEFAULT_METASTORE_TABLE);
        if (!admin.tableExists(metaTable)) {
            // if meta table don't exists just create it.
            HTableDescriptor desc = new HTableDescriptor(metaTable);
            HColumnDescriptor family = new HColumnDescriptor(FConstants.CATALOG_FAMILY_STR);
            family.setInMemory(true);
            desc.addFamily(family);
            admin.createTable(desc);
        }
        admin.close();
    } catch (ZooKeeperConnectionException zce) {
        throw new MetaException("FMeta could not connect to zookeeper ", zce);
    } catch (MasterNotRunningException mnre) {
        throw new MetaException("FMeta Could not connect to HBase Master ", mnre);
    } catch (IOException e) {
        throw new MetaException(e);
    }
}

---

From source file:com.alibaba.wasp.meta.FMetaUtil.java

License:Apache License

/**
 * Use HBaseAdmin to delete the meta table .
 */// w  w  w . ja v a  2s .c  o m
public static boolean clean(Configuration conf) {
    try {
        HBaseAdmin admin = new HBaseAdmin(conf);
        String metaTable = conf.get(FConstants.METASTORE_TABLE, FConstants.DEFAULT_METASTORE_TABLE);
        if (admin.tableExists(metaTable)) {
            admin.disableTable(metaTable);
            admin.deleteTable(metaTable);
        }
        admin.close();
        return true;
    } catch (ZooKeeperConnectionException zce) {
        LOG.error("Could not connect to zookeeper ", zce);
    } catch (MasterNotRunningException mnre) {
        LOG.error("Could not connect to HBase Master ", mnre);
    } catch (IOException e) {
        LOG.error(e);
    }
    return false;
}

---

From source file:com.alibaba.wasp.util.Utils.java

License:Apache License

/**
 * Get the system property. If the system property is not set, or if a
 * security exception occurs, the default value is returned.
 * //from ww w. j a  v  a2s.  c  o  m
 * @param key
 *          the key
 * @param defaultValue
 *          the default value
 * @return the value
 */
public static String getProperty(Configuration conf, String key, String defaultValue) {
    try {
        return conf.get(key, defaultValue);
    } catch (SecurityException se) {
        return defaultValue;
    }
}

---

From source file:com.alibaba.wasp.zookeeper.FQuorumPeer.java

License:Apache License

static void writeMyID(Configuration conf, Properties properties) throws IOException {
    long myId = -1;
    String myAddress = Strings/*from w w  w. j  a  v  a2s . co m*/
            .domainNamePointerToHostName(DNS.getDefaultHost(conf.get("wasp.zookeeper.dns.interface", "default"),
                    conf.get("wasp.zookeeper.dns.nameserver", "default")));

    List<String> ips = new ArrayList<String>();

    // Add what could be the best (configured) match
    ips.add(myAddress.contains(".") ? myAddress : StringUtils.simpleHostname(myAddress));

    // For all nics get all hostnames and IPs
    Enumeration<?> nics = NetworkInterface.getNetworkInterfaces();
    while (nics.hasMoreElements()) {
        Enumeration<?> rawAdrs = ((NetworkInterface) nics.nextElement()).getInetAddresses();
        while (rawAdrs.hasMoreElements()) {
            InetAddress inet = (InetAddress) rawAdrs.nextElement();
            ips.add(StringUtils.simpleHostname(inet.getHostName()));
            ips.add(inet.getHostAddress());
        }
    }

    for (Entry<Object, Object> entry : properties.entrySet()) {
        String key = entry.getKey().toString().trim();
        String value = entry.getValue().toString().trim();
        if (key.startsWith("server.")) {
            int dot = key.indexOf('.');
            long id = Long.parseLong(key.substring(dot + 1));
            String[] parts = value.split(":");
            String address = parts[0];
            if (addressIsLocalHost(address) || ips.contains(address)) {
                myId = id;
                break;
            }
        }
    }

    // Set the max session timeout from the provided client-side timeout
    properties.setProperty("maxSessionTimeout", conf.get("zookeeper.session.timeout", "180000"));

    if (myId == -1) {
        throw new IOException(
                "Could not find my address: " + myAddress + " in list of ZooKeeper quorum servers");
    }

    String dataDirStr = properties.get("dataDir").toString().trim();
    File dataDir = new File(dataDirStr);
    if (!dataDir.isDirectory()) {
        if (!dataDir.mkdirs()) {
            throw new IOException("Unable to create data dir " + dataDir);
        }
    }

    File myIdFile = new File(dataDir, "myid");
    PrintWriter w = new PrintWriter(myIdFile);
    w.println(myId);
    w.close();
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• »