Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.aliuge.crawler.jobconf; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.HashMap; import org.aliuge.crawler.exception.ConfigurationException; import org.aliuge.crawler.exception.StorageException; import org.aliuge.crawler.store.StorageType; import org.aliuge.crawler.util.EncryptUtils; import org.aliuge.crawler.util.MD5Utils; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; /** * ?es * * @author * @date 2015-8-3 * @desc */ public class StoreConfig extends Configuration { private static Logger log = Logger.getLogger(StoreConfig.class); private String type = "default"; private IDPolicy id = IDPolicy.auto; private String policyRef = ""; private int threadNum = 2; private String pluginClass = null; private HBaseConfig hconfig = null; private MongodbConfig mongodbConfig = null; public StoreConfig() { } public StoreConfig loadConfig(Document confDoc) throws StorageException { Document doc = confDoc; super.setJobName(doc.select("job").attr("name")); super.setIndexName(doc.select("job").attr("indexName")); Elements e = doc.select("store"); this.type = e.select("type").text(); if (StringUtils.isNotBlank(e.select("threadNum").text())) { this.threadNum = Integer.parseInt(e.select("threadNum").text()); } String className = e.select("plugin").text(); if (StringUtils.isNotBlank(className)) { this.pluginClass = className; } if (!StorageType.containsValue(this.type)) { log.info("?" + this.type); throw new StorageException( "???store? mysql,hbase,elasticSearch,localFile,mongodb"); } if (this.type.equalsIgnoreCase(StorageType.hbase.getValue())) { String tName = e.select("table").first().attr("name"); String fName = e.select("family").first().text(); this.hconfig = new HBaseConfig(tName, fName); } else if (this.type.equalsIgnoreCase(StorageType.mongodb.getValue())) { String dbName = e.select("db").first().attr("name"); String collection = e.select("collection").first().text(); String port = e.select("port").text(); String host = e.select("host").text(); this.mongodbConfig = new MongodbConfig(dbName, collection, host, port); } else if (this.type.equalsIgnoreCase(StorageType.mysql.getValue())) { } else if (this.type.equalsIgnoreCase(StorageType.elasticsearch.getValue())) { } // id? String idPolicy = e.select("idPolicy").text(); if (StringUtils.isNotBlank(idPolicy)) { id = EnumUtils.getEnum(IDPolicy.class, idPolicy); if (!IDPolicy.auto.equals(id)) { String pref = e.select("ref").text(); if (StringUtils.isNotBlank(pref)) { this.policyRef = pref; } if (StringUtils.isBlank(this.policyRef)) { try { throw new ConfigurationException("ID??"); } catch (Exception e2) { e2.printStackTrace(); } } } } return this; } /** * ??ID?ID * * @return */ public String genId(HashMap<String, Object> data) { switch (id) { case auto: return null; case md5: return MD5Utils.createMD5((String) data.get(policyRef)); case base64: return EncryptUtils.encodeBase64((String) data.get(policyRef)); case urlencode: try { return URLEncoder.encode((String) data.get(policyRef), "UTF-8"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } default: return null; } } public String getType() { return type; } public void setType(String type) { this.type = type; } public int getThreadNum() { return threadNum; } public void setThreadNum(int threadNum) { this.threadNum = threadNum; } public String getPluginClass() { return pluginClass; } public void setPluginClass(String pluginClass) { this.pluginClass = pluginClass; } public HBaseConfig getHconfig() { return hconfig; } public void setHconfig(HBaseConfig hconfig) { this.hconfig = hconfig; } public MongodbConfig getMongodbConfig() { return mongodbConfig; } public void setMongodbConfig(MongodbConfig mongodbConfig) { this.mongodbConfig = mongodbConfig; } @Override public String toString() { return "StoreConfig [type=" + type + ", threadNum=" + threadNum + ", pluginClass=" + pluginClass + ", hconfig=" + hconfig + "]"; } /** * @author shenbaise * @date 2014310 desc:id? */ public enum IDPolicy { auto, md5, base64, urlencode } public class HBaseConfig { private String tableName; private String familyName; public HBaseConfig() { super(); } public HBaseConfig(String tableName, String familyName) { super(); this.tableName = tableName; this.familyName = familyName; } public String getTableName() { return tableName; } public void setTableName(String tableName) { this.tableName = tableName; } public String getFamilyName() { return familyName; } public void setFamilyName(String familyName) { this.familyName = familyName; } @Override public String toString() { return "HBaseConfig [tableName=" + tableName + ", familyName=" + familyName + "]"; } } public class MongodbConfig { private String database; private String collection; private String host; private String port; public MongodbConfig() { super(); } public MongodbConfig(String database, String collection, String host, String port) { super(); this.database = database; this.collection = collection; this.host = host; this.port = port; } public String getDatabase() { return database; } public void setDatabase(String database) { this.database = database; } public String getCollection() { return collection; } public void setCollection(String collection) { this.collection = collection; } public String getHost() { return host; } public void setHost(String host) { this.host = host; } public String getPort() { return port; } public void setPort(String port) { this.port = port; } @Override public String toString() { return "MongodbConfig [database=" + database + ", collection=" + collection + "]"; } } }