Example usage for org.apache.hadoop.conf Configuration get

Introduction

In this page you can find the example usage for org.apache.hadoop.conf Configuration get.

Prototype

public String get(String name) 

Source Link

Document

Get the value of the name property, null if no such property exists.

Usage

From source file:com.chinamobile.bcbsp.bspcontroller.BSPController.java

License:Apache License

/**
 * get BSPcontroller socket address./*from   w w  w .  j av  a2s.  c om*/
 * @param conf
 *        BSP conf to get the address.
 * @return
 *        socket address of controller.
 */
public static InetSocketAddress getAddress(Configuration conf) {
    String bspControllerStr = conf.get(Constants.BC_BSP_CONTROLLER_ADDRESS);
    return NetUtils.createSocketAddr(bspControllerStr);
}

---

From source file:com.chinamobile.bcbsp.bspcontroller.JobInProgress.java

License:Apache License

/**
 * clean checkpoint and delete the checkpoint information
 * @return chean result./*from   w  w  w .j  a  va  2  s.c  o m*/
 */
private boolean cleanCheckpoint() {
    if (job.getCheckpointType().equals("HBase")) {
        String tableName = null;
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.property.clientPort", job.getConf().get(Constants.ZOOKEPER_CLIENT_PORT));
        conf.set("hbase.zookeeper.quorum", "master");
        conf.set("hbase.master", "master:60000");
        HBaseAdmin admin;
        try {
            admin = new HBaseAdmin(conf);
            for (int i = 0; i < this.attemptIDList.size(); i++) {
                tableName = this.attemptIDList.get(i).toString();
                if (admin.tableExists(tableName)) {
                    admin.disableTable(tableName);
                    admin.deleteTable(tableName);
                }
            }
            admin.close();
            return true;
        } catch (MasterNotRunningException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        } catch (ZooKeeperConnectionException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        } catch (IOException e) {
            LOG.error("Exception has happened and been catched!", e);
            e.printStackTrace();
        }
        return false;
    } else if (job.getCheckpointType().equals("HDFS")) {
        try {
            String uri = conf.get(Constants.BC_BSP_CHECKPOINT_WRITEPATH) + "/" + job.getJobID().toString()
                    + "/";
            // FileSystem fs = FileSystem.get(URI.create(uri), conf);
            BSPFileSystem bspfs = new BSPFileSystemImpl(URI.create(uri), conf);
            // if(fs.exists(new Path(uri))) {
            // fs.delete(new Path(uri), true);
            // }
            if (bspfs.exists(new BSPHdfsImpl().newPath(uri))) {
                bspfs.delete(new BSPHdfsImpl().newPath(uri), true);
            }
            return true;
        } catch (IOException e) {
            LOG.error("Exception has happened and been catched!", e);
            return false;
        }
    }
    return false;
}

---

From source file:com.chinamobile.bcbsp.fault.storage.MonitorFaultLog.java

License:Apache License

/**
 * get the hdfs namenode hostname/*from   w  w w . j a va 2s  . com*/
 * @return hdfsNamenodehostName
 */
private String getHdfsNameNodeHostName() {
    Configuration conf = new Configuration(false);
    String HADOOP_HOME = System.getenv("HADOOP_HOME");
    String corexml = HADOOP_HOME + "/conf/core-site.xml";
    conf.addResource(new Path(corexml));
    String hdfsNamenodehostName = conf.get("fs.default.name");
    return hdfsNamenodehostName;
}

---

From source file:com.chinamobile.bcbsp.io.BSPFileInputFormat.java

License:Apache License

/**
 * Add a {@link Path} to the list of inputs for the BC_BSP job.
 *
 * @param job/*from   www.j a  va2 s .  c o  m*/
 *        the current job BSPJob.
 * @param path
 *        {@link Path} to be added to the list of inputs for the BC_BSP job.
 */
public static void addInputPath(BSPJob job, Path path) throws IOException {
    Configuration conf = job.getConf();
    FileSystem fs = FileSystem.get(conf);
    path = path.makeQualified(fs);
    String dirStr = StringUtils.escapeString(path.toString());
    String dirs = conf.get(Constants.USER_BC_BSP_JOB_INPUT_DIR);
    conf.set(Constants.USER_BC_BSP_JOB_INPUT_DIR, dirs == null ? dirStr : dirs + "," + dirStr);
}

---

From source file:com.chinamobile.bcbsp.io.db.TableInputFormat.java

License:Apache License

@Override
public void initialize(Configuration configuration) {
    this.conf = HBaseConfiguration.create(configuration);
    this.conf.set("hbase.master", configuration.get("hbase.master"));
    this.conf.set("hbase.zookeeper.quorum", configuration.get("hbase.zookeeper.quorum"));
    //this.conf.set(INPUT_TABLE, configuration.get(INPUT_TABLE));
    this.conf.set(SCAN_COLUMN_FAMILY, configuration.get(SCAN_COLUMN_FAMILY));
    String tableName = conf.get(INPUT_TABLE);
    try {//from   ww w  .java 2s .c  om
        setTable(new HTable(conf, tableName));
    } catch (Exception e) {
        LOG.error(StringUtils.stringifyException(e));
    }
    Scan scan = null;
    try {
        scan = new Scan();
        if (conf.get(SCAN_COLUMNS) != null) {
            //scan.addColumns(conf.get(SCAN_COLUMNS));    
            scan.addColumn(Bytes.toBytes(conf.get(SCAN_COLUMN_FAMILY)), Bytes.toBytes(conf.get(SCAN_COLUMNS)));
        }
        if (conf.get(SCAN_COLUMN_FAMILY) != null) {
            scan.addFamily(Bytes.toBytes(conf.get(SCAN_COLUMN_FAMILY)));
        }
        if (conf.get(SCAN_TIMESTAMP) != null) {
            scan.setTimeStamp(Long.parseLong(conf.get(SCAN_TIMESTAMP)));
        }
        if (conf.get(SCAN_TIMERANGE_START) != null && conf.get(SCAN_TIMERANGE_END) != null) {
            scan.setTimeRange(Long.parseLong(conf.get(SCAN_TIMERANGE_START)),
                    Long.parseLong(conf.get(SCAN_TIMERANGE_END)));
        }
        if (conf.get(SCAN_MAXVERSIONS) != null) {
            scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
        }
        if (conf.get(SCAN_CACHEDROWS) != null) {
            scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
        }
        // false by default, full table scans generate too much BC churn
        scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
    } catch (Exception e) {
        LOG.error(StringUtils.stringifyException(e));
    }
    setScan(scan);
}

---

From source file:com.chinamobile.bcbsp.io.db.TableOutputFormat.java

License:Apache License

@Override
public void initialize(Configuration otherConf) {
    this.conf = HBaseConfiguration.create(otherConf);
    this.conf.set("hbase.master", otherConf.get("hbase.master"));
    this.conf.set("hbase.zookeeper.quorum", otherConf.get("hbase.zookeeper.quorum"));
    String tableName = this.conf.get(OUTPUT_TABLE);
    String address = this.conf.get(QUORUM_ADDRESS);
    String serverClass = this.conf.get(REGION_SERVER_CLASS);
    String serverImpl = this.conf.get(REGION_SERVER_IMPL);
    try {/*from   w  w w .j a  va 2  s  .  c o  m*/
        if (address != null) {
            ZKUtil.applyClusterKeyToConf(this.conf, address);
        }
        if (serverClass != null) {
            this.conf.set(HConstants.REGION_SERVER_CLASS, serverClass);
            this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
        }
        HTable table = new HTable(this.conf, tableName);
        table.setAutoFlush(false);
        setTable(table);
        LOG.info("Created table instance for " + tableName);
    } catch (IOException e) {
        LOG.error("[initialize]", e);
    }
}

---

From source file:com.chinamobile.bcbsp.pipes.Application.java

License:Apache License

/**
 * This method is the constructor./*from www.  j  av  a2  s .  c  o m*/
 * @param conf
 *        contains Job configuration
 * @param processType
 *        the type of c++ process(for staff or workmanager)
 */
public Application(Configuration conf, String processType) throws IOException, InterruptedException {
    serverSocket = new ServerSocket(0);
    Map<String, String> env = new HashMap<String, String>();
    env.put("TMPDIR", System.getProperty("java.io.tmpdir"));
    env.put("bcbsp.pipes.command.port", Integer.toString(serverSocket.getLocalPort()));
    env.put("processType", processType);
    List<String> cmd = new ArrayList<String>();
    String executable = conf.get(Constants.USER_BC_BSP_JOB_EXE);
    FileUtil.chmod(executable, "a+x");
    cmd.add(executable);
    process = runClient(cmd, env);
    clientSocket = serverSocket.accept();
    this.handler = new TaskHandler();
    this.downlink = new BinaryProtocol(clientSocket, handler);
    this.downlink.start();
}

---

From source file:com.chinamobile.bcbsp.workermanager.WorkerManager.java

License:Apache License

/**
 * Constructor.//ww w  .j a va  2  s.  c  o  m
 * @param conf Configuration
 */
public WorkerManager(Configuration conf) throws IOException {
    LOG.info("worker start");
    this.conf = conf;
    String mode = conf.get(Constants.BC_BSP_CONTROLLER_ADDRESS);
    if (!mode.equals("local")) {
        choseActiveControllerAddress();
    }
}

---

From source file:com.chinnu.churndetection.fuzzykmeans.FuzzyKMeansMapper.java

@Override
protected void setup(Mapper<LongWritable, Text, IntWritable, Vector>.Context context)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();

    CENTERS = conf.get(Constants.CENTER_TEXT);
    STARTINDEX = conf.getInt(Constants.STARTINDEX, 0);
    ENDINDEX = conf.getInt(Constants.ENDINDEX, 0);
    CLASSINDEX = conf.getInt(Constants.CLASSINDEX, 0);
    DATALENGTH = ENDINDEX - STARTINDEX;/*  w  w  w .ja  va  2 s . c om*/
    super.setup(context);
}

---

From source file:com.chinnu.churndetection.fuzzykmeans.FuzzyKMeansReducer.java

@Override
protected void setup(Reducer<IntWritable, Vector, IntWritable, Text>.Context context)
        throws IOException, InterruptedException {

    Configuration conf = context.getConfiguration();

    NEW_CENTER = conf.get(Constants.NEXTCENTER);
    CURR_CENTER = conf.get(Constants.CENTER_TEXT);
    STARTINDEX = conf.getInt(Constants.STARTINDEX, 0);
    ENDINDEX = conf.getInt(Constants.ENDINDEX, 0);
    DATALENGTH = ENDINDEX - STARTINDEX;/* ww w.java  2  s.  co m*/

    super.setup(context);
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• »