List of usage examples for org.apache.hadoop.conf Configuration getStrings
public String[] getStrings(String name)
name property as an array of Strings. From source file:com.linkedin.thirdeye.hadoop.derivedcolumn.transformation.DerivedColumnTransformationTest.java
License:Apache License
private void resetAvroSerialization() throws IOException { Configuration conf = mapDriver.getConfiguration(); conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization"); Schema outputSchema = new Schema.Parser() .parse(ClassLoader.getSystemResourceAsStream(TRANSFORMATION_SCHEMA)); String[] currentSerializations = conf.getStrings(HADOOP_IO_SERIALIZATION); String[] finalSerializations = new String[currentSerializations.length + 1]; System.arraycopy(currentSerializations, 0, finalSerializations, 0, currentSerializations.length); finalSerializations[finalSerializations.length - 1] = AvroSerialization.class.getName(); mapDriver.getConfiguration().setStrings(HADOOP_IO_SERIALIZATION, finalSerializations); AvroSerialization.addToConfiguration(conf); AvroSerialization.setKeyWriterSchema(conf, outputSchema); AvroSerialization.setValueWriterSchema(conf, Schema.create(Schema.Type.NULL)); }
From source file:com.marklogic.contentpump.ContentWithFileNameWritable.java
License:Apache License
@Override public Content getContent(Configuration conf, ContentCreateOptions options, String uri) { String[] collections = conf.getStrings(MarkLogicConstants.OUTPUT_COLLECTION); String collectionUri = null;/*from w ww. j a v a2s. c o m*/ try { URI fileUri = new URI(null, null, null, 0, fileName, null, null); collectionUri = fileUri.toString(); } catch (URISyntaxException e) { LOG.error("Error parsing file name as URI " + fileName, e); } if (collections != null) { List<String> optionList = new ArrayList<String>(); Collections.addAll(optionList, collections); if (collectionUri != null) { optionList.add(collectionUri); } collections = optionList.toArray(new String[0]); for (int i = 0; i < collections.length; i++) { collections[i] = collections[i].trim(); } options.setCollections(collections); } else { String[] col = new String[1]; col[0] = collectionUri; options.setCollections(col); } Content content = null; if (value instanceof Text) { content = ContentFactory.newContent(uri, ((Text) value).toString(), options); } else if (value instanceof MarkLogicNode) { content = ContentFactory.newContent(uri, ((MarkLogicNode) value).get(), options); } else if (value instanceof BytesWritable) { content = ContentFactory.newContent(uri, ((BytesWritable) value).getBytes(), 0, ((BytesWritable) value).getLength(), options); } return content; }
From source file:com.marklogic.contentpump.RDFWritable.java
License:Apache License
@Override public Content getContent(Configuration conf, ContentCreateOptions options, String uri) { String[] collections = conf.getStrings(MarkLogicConstants.OUTPUT_COLLECTION); String outputGraph = conf.get(MarkLogicConstants.OUTPUT_GRAPH); String outputOverrideGraph = conf.get(MarkLogicConstants.OUTPUT_OVERRIDE_GRAPH); if (collections != null) { List<String> optionList = new ArrayList<String>(); if (graphUri == null) { //no graph specified in quad if (outputGraph != null)//output_graph is set optionList.add(outputGraph.trim()); else if (outputOverrideGraph != null) { optionList.add(outputOverrideGraph.trim()); }//from www. j a v a 2 s . c o m } else { if (outputOverrideGraph != null) optionList.add(outputOverrideGraph); else optionList.add(graphUri);//use quad's graph } //collections are always added Collections.addAll(optionList, collections); collections = optionList.toArray(new String[0]); for (int i = 0; i < collections.length; i++) { collections[i] = collections[i].trim(); } options.setCollections(collections); } else { if (graphUri == null) { if (outputOverrideGraph != null) { graphUri = outputOverrideGraph; } else if (outputGraph != null) { graphUri = outputGraph; } else { graphUri = "http://marklogic.com/semantics#default-graph"; } } String[] col = new String[1]; col[0] = graphUri; options.setCollections(col); } options.setGraph(graphUri); //permissions if (permissions != null) options.setPermissions(permissions); Content content = null; if (value instanceof Text) { content = ContentFactory.newContent(uri, ((Text) value).toString(), options); } else if (value instanceof MarkLogicNode) { content = ContentFactory.newContent(uri, ((MarkLogicNode) value).get(), options); } else if (value instanceof BytesWritable) { content = ContentFactory.newContent(uri, ((BytesWritable) value).getBytes(), 0, ((BytesWritable) value).getLength(), options); } return content; }
From source file:com.marklogic.mapreduce.ContentOutputFormat.java
License:Apache License
@Override public void checkOutputSpecs(Configuration conf, ContentSource cs) throws IOException { Session session = null;// w w w . ja va 2 s . co m ResultSequence result = null; try { session = cs.newSession(); RequestOptions options = new RequestOptions(); options.setDefaultXQueryVersion("1.0-ml"); session.setDefaultRequestOptions(options); // clear output dir if specified String outputDir = conf.get(OUTPUT_DIRECTORY); if (outputDir != null) { outputDir = outputDir.endsWith("/") ? outputDir : outputDir + "/"; if (conf.getBoolean(OUTPUT_CLEAN_DIR, false)) { // delete directory if exists String queryText = DELETE_DIRECTORY_TEMPLATE.replace(DIRECTORY_TEMPLATE, outputDir); AdhocQuery query = session.newAdhocQuery(queryText); result = session.submitRequest(query); } else { // ensure nothing exists under output dir String queryText = CHECK_DIRECTORY_EXIST_TEMPLATE.replace(DIRECTORY_TEMPLATE, outputDir); AdhocQuery query = session.newAdhocQuery(queryText); result = session.submitRequest(query); if (result.hasNext()) { ResultItem item = result.next(); if (((XSBoolean) (item.getItem())).asBoolean()) { throw new IllegalStateException("Directory " + outputDir + " already exists"); } } else { throw new IllegalStateException("Failed to query directory content."); } } } // initialize server host name and assignment policy initialize(session); // ensure manual directory creation if (fastLoad) { LOG.info("Running in fast load mode"); // store forest-info map into config system DefaultStringifier.store(conf, queryForestInfo(cs), OUTPUT_FOREST_HOST); AdhocQuery query = session.newAdhocQuery(DIRECTORY_CREATE_QUERY); result = session.submitRequest(query); if (result.hasNext()) { ResultItem item = result.next(); String dirMode = item.asString(); if (!dirMode.equals(MANUAL_DIRECTORY_MODE)) { throw new IllegalStateException("Manual directory creation mode is required. " + "The current creation mode is " + dirMode + "."); } } else { throw new IllegalStateException("Failed to query directory creation mode."); } } else { TextArrayWritable hostArray; // 23798: replace hostname in forest config with // user-specified output host String outputHost = conf.get(OUTPUT_HOST); if (MODE_LOCAL.equals(conf.get(EXECUTION_MODE))) { hostArray = queryHosts(cs, initHostName, outputHost); } else { hostArray = queryHosts(cs); } DefaultStringifier.store(conf, hostArray, OUTPUT_FOREST_HOST); } // validate capabilities String[] perms = conf.getStrings(OUTPUT_PERMISSION); if (perms != null && perms.length > 0) { if (perms.length % 2 != 0) { throw new IllegalStateException("Permissions are expected to be in <role, capability> pairs."); } int i = 0; while (i + 1 < perms.length) { String roleName = perms[i++]; if (roleName == null || roleName.isEmpty()) { throw new IllegalStateException("Illegal role name: " + roleName); } String perm = perms[i].trim(); if (!perm.equalsIgnoreCase(ContentCapability.READ.toString()) && !perm.equalsIgnoreCase(ContentCapability.EXECUTE.toString()) && !perm.equalsIgnoreCase(ContentCapability.INSERT.toString()) && !perm.equalsIgnoreCase(ContentCapability.UPDATE.toString())) { throw new IllegalStateException("Illegal capability: " + perm); } i++; } } } catch (RequestException ex) { throw new IOException(ex); } finally { if (session != null) { session.close(); } if (result != null) { result.close(); } } }
From source file:com.marklogic.mapreduce.ContentWriter.java
License:Apache License
public ContentWriter(Configuration conf, Map<String, ContentSource> forestSourceMap, boolean fastLoad, AssignmentManager am) {//from ww w. j a v a 2 s . c om super(conf, null); this.fastLoad = fastLoad; this.forestSourceMap = forestSourceMap; this.am = am; permsMap = new HashMap<String, ContentPermission[]>(); int srcMapSize = forestSourceMap.size(); forestIds = new String[srcMapSize]; // key order in key set is guaranteed by LinkedHashMap, // i.e., the order keys are inserted forestIds = forestSourceMap.keySet().toArray(forestIds); hostId = (int) (Math.random() * srcMapSize); // arraySize is the number of forests in fast load mode; 1 otherwise. int arraySize = fastLoad ? srcMapSize : 1; sessions = new Session[arraySize]; stmtCounts = new int[arraySize]; outputDir = conf.get(OUTPUT_DIRECTORY); batchSize = conf.getInt(BATCH_SIZE, DEFAULT_BATCH_SIZE); pendingUris = new HashMap[arraySize]; for (int i = 0; i < arraySize; i++) { pendingUris[i] = new HashMap<Content, DocumentURI>(); } if (fastLoad && (am.getPolicy().getPolicyKind() == AssignmentPolicy.Kind.STATISTICAL || am.getPolicy().getPolicyKind() == AssignmentPolicy.Kind.RANGE)) { countBased = true; if (batchSize > 1) { forestContents = new Content[1][batchSize]; counts = new int[1]; } sfId = -1; } else { if (batchSize > 1) { forestContents = new Content[arraySize][batchSize]; counts = new int[arraySize]; } sfId = 0; } String[] perms = conf.getStrings(OUTPUT_PERMISSION); List<ContentPermission> permissions = null; if (perms != null && perms.length > 0) { int i = 0; while (i + 1 < perms.length) { String roleName = perms[i++]; if (roleName == null || roleName.isEmpty()) { LOG.error("Illegal role name: " + roleName); continue; } String perm = perms[i].trim(); ContentCapability capability = null; if (perm.equalsIgnoreCase(ContentCapability.READ.toString())) { capability = ContentCapability.READ; } else if (perm.equalsIgnoreCase(ContentCapability.EXECUTE.toString())) { capability = ContentCapability.EXECUTE; } else if (perm.equalsIgnoreCase(ContentCapability.INSERT.toString())) { capability = ContentCapability.INSERT; } else if (perm.equalsIgnoreCase(ContentCapability.UPDATE.toString())) { capability = ContentCapability.UPDATE; } else { LOG.error("Illegal permission: " + perm); } if (capability != null) { if (permissions == null) { permissions = new ArrayList<ContentPermission>(); } permissions.add(new ContentPermission(capability, roleName)); } i++; } } options = new ContentCreateOptions(); String[] collections = conf.getStrings(OUTPUT_COLLECTION); if (collections != null) { for (int i = 0; i < collections.length; i++) { collections[i] = collections[i].trim(); } options.setCollections(collections); } options.setQuality(conf.getInt(OUTPUT_QUALITY, 0)); if (permissions != null) { options.setPermissions(permissions.toArray(new ContentPermission[permissions.size()])); } String contentTypeStr = conf.get(CONTENT_TYPE, DEFAULT_CONTENT_TYPE); ContentType contentType = ContentType.valueOf(contentTypeStr); if (contentType == ContentType.UNKNOWN) { formatNeeded = true; } else { options.setFormat(contentType.getDocumentFormat()); } options.setLanguage(conf.get(OUTPUT_CONTENT_LANGUAGE)); String repairLevel = conf.get(OUTPUT_XML_REPAIR_LEVEL, DEFAULT_OUTPUT_XML_REPAIR_LEVEL).toLowerCase(); options.setNamespace(conf.get(OUTPUT_CONTENT_NAMESPACE)); if (DocumentRepairLevel.DEFAULT.toString().equals(repairLevel)) { options.setRepairLevel(DocumentRepairLevel.DEFAULT); } else if (DocumentRepairLevel.NONE.toString().equals(repairLevel)) { options.setRepairLevel(DocumentRepairLevel.NONE); } else if (DocumentRepairLevel.FULL.toString().equals(repairLevel)) { options.setRepairLevel(DocumentRepairLevel.FULL); } streaming = conf.getBoolean(OUTPUT_STREAMING, false); tolerateErrors = conf.getBoolean(OUTPUT_TOLERATE_ERRORS, false); String encoding = conf.get(MarkLogicConstants.OUTPUT_CONTENT_ENCODING); if (encoding != null) { options.setEncoding(encoding); } options.setTemporalCollection(conf.get(TEMPORAL_COLLECTION)); needCommit = txnSize > 1 || (batchSize > 1 && tolerateErrors); if (needCommit) { commitUris = new ArrayList[arraySize]; for (int i = 0; i < arraySize; i++) { commitUris[i] = new ArrayList<DocumentURI>(txnSize * batchSize); } } }
From source file:com.marklogic.mapreduce.utilities.URIUtil.java
License:Apache License
/** * Apply URI replacement configuration option to a URI source string. The * configuration option is a list of comma separated pairs of regex * patterns and replacements. Validation of the configuration is done at * command parsing time./* w ww . ja v a 2s .c o m*/ * * @param uriSource * @param conf * @return result URI string */ public static String applyUriReplace(String uriSource, Configuration conf) { if (uriSource == null) return null; String[] uriReplace = conf.getStrings(CONF_OUTPUT_URI_REPLACE); if (uriReplace == null) return uriSource; for (int i = 0; i < uriReplace.length - 1; i += 2) { String replacement = uriReplace[i + 1].trim(); replacement = replacement.substring(1, replacement.length() - 1); uriSource = uriSource.replaceAll(uriReplace[i], replacement); } return uriSource; }
From source file:com.moz.fiji.hadoop.configurator.ConfigurationMethod.java
License:Apache License
/** * Calls an object's method with the value read from a Configuration instance. * * @param instance The object to populate. * @param conf The configuration to read from. * @throws IllegalAccessException If the method cannot be called on the object. * @throws HadoopConfigurationException If there is a problem with the annotation definition. *///from w w w . ja v a2 s . c om public void call(Object instance, Configuration conf) throws IllegalAccessException { final String key = getKey(); if (null == key) { throw new HadoopConfigurationException("Missing 'key' attribute of @HadoopConf on " + instance.getClass().getName() + "." + mMethod.getName()); } if (!mMethod.isAccessible()) { mMethod.setAccessible(true); } final Class<?>[] parameterTypes = mMethod.getParameterTypes(); if (1 != parameterTypes.length) { throw new HadoopConfigurationException( "Methods annotated with @HadoopConf must have exactly one parameter: " + instance.getClass().getName() + "." + mMethod.getName()); } final Class<?> parameterType = parameterTypes[0]; try { try { if (boolean.class == parameterType) { mMethod.invoke(instance, conf.getBoolean(key, Boolean.parseBoolean(getDefault()))); } else if (float.class == parameterType) { mMethod.invoke(instance, conf.getFloat(key, Float.parseFloat(getDefault()))); } else if (double.class == parameterType) { mMethod.invoke(instance, conf.getFloat(key, Float.parseFloat(getDefault()))); } else if (int.class == parameterType) { mMethod.invoke(instance, conf.getInt(key, Integer.parseInt(getDefault()))); } else if (long.class == parameterType) { mMethod.invoke(instance, conf.getLong(key, Long.parseLong(getDefault()))); } else if (parameterType.isAssignableFrom(String.class)) { mMethod.invoke(instance, conf.get(key, getDefault())); } else if (parameterType.isAssignableFrom(Collection.class)) { mMethod.invoke(instance, conf.getStringCollection(key)); } else if (String[].class == parameterType) { mMethod.invoke(instance, new Object[] { conf.getStrings(key) }); } else { throw new HadoopConfigurationException( "Unsupported method parameter type annotated by @HadoopConf: " + instance.getClass().getName() + "." + mMethod.getName()); } } catch (NumberFormatException e) { mMethod.invoke(instance, getDefault()); } } catch (InvocationTargetException e) { throw new HadoopConfigurationException(e); } }
From source file:com.moz.fiji.hadoop.configurator.ConfigurationVariable.java
License:Apache License
/** * Populates an object's field with the value read from a Configuration instance. * * @param instance The object to populate. * @param conf The configuration to read from. * @throws IllegalAccessException If the field cannot be set on the object. * @throws HadoopConfigurationException If there is a problem with the annotation definition. */// www . jav a2s . co m public void setValue(Object instance, Configuration conf) throws IllegalAccessException { final String key = getKey(); if (null == key) { throw new HadoopConfigurationException("Missing 'key' attribute of @HadoopConf on " + instance.getClass().getName() + "." + mField.getName()); } if (null == conf.get(key) && mAnnotation.defaultValue().isEmpty()) { // Nothing set in the configuration, and no default value // specified. Just leave the field alone. return; } if (!mField.isAccessible()) { mField.setAccessible(true); } try { if (boolean.class == mField.getType()) { mField.setBoolean(instance, conf.getBoolean(key, getDefaultBoolean(instance))); } else if (float.class == mField.getType()) { mField.setFloat(instance, conf.getFloat(key, getDefaultFloat(instance))); } else if (double.class == mField.getType()) { mField.setDouble(instance, conf.getFloat(key, getDefaultDouble(instance))); } else if (int.class == mField.getType()) { mField.setInt(instance, conf.getInt(key, getDefaultInt(instance))); } else if (long.class == mField.getType()) { mField.setLong(instance, conf.getLong(key, getDefaultLong(instance))); } else if (mField.getType().isAssignableFrom(String.class)) { mField.set(instance, conf.get(key, getDefaultString(instance))); } else if (mField.getType().isAssignableFrom(Collection.class)) { mField.set(instance, conf.getStringCollection(key)); } else if (String[].class == mField.getType()) { mField.set(instance, conf.getStrings(key)); } else { throw new HadoopConfigurationException("Unsupported field type annotated by @HadoopConf: " + instance.getClass().getName() + "." + mField.getName()); } } catch (NumberFormatException e) { // That's okay. The default value for the field will be kept. } }
From source file:com.moz.fiji.mapreduce.kvstore.TestKeyValueStoreConfiguration.java
License:Apache License
@Test public void testStoreStringArray() { Configuration parent = new Configuration(false); KeyValueStoreConfiguration isolated = KeyValueStoreConfiguration.createInConfiguration(parent, 0); isolated.setStrings("foo-key", "first", "second", "third"); assertTrue(Arrays.equals(new String[] { "first", "second", "third" }, isolated.getStrings("foo-key"))); // Also test the specifying a defaultValue works. assertTrue(Arrays.equals(new String[] { "first", "second", "third" }, isolated.getStrings("foo-key", new String[0]))); // Check that this value is stored in the namespace on the parent: Configuration delegate = isolated.getDelegate(); assertTrue(Arrays.equals(new String[] { "first", "second", "third" }, delegate.getStrings(KeyValueStoreConfiguration.confKeyAtIndex("foo-key", 0)))); }
From source file:com.nearinfinity.honeycomb.hbase.bulkload.BulkLoadMapper.java
License:Apache License
@Override protected void setup(Context context) throws IOException { Configuration conf = context.getConfiguration(); char separator = conf.get(SEPARATOR, " ").charAt(0); columns = conf.getStrings(SQL_COLUMNS); String sqlTable = conf.get(SQL_TABLE); String hbaseTable = conf.get(ConfigConstants.TABLE_NAME); String columnFamily = conf.get(ConfigConstants.COLUMN_FAMILY); // Check that necessary configuration variables are set checkNotNull(conf.get(HConstants.ZOOKEEPER_QUORUM), HConstants.ZOOKEEPER_QUORUM + NOT_SET_ERROR); checkNotNull(sqlTable, SQL_TABLE + NOT_SET_ERROR); checkNotNull(columns, SQL_COLUMNS + NOT_SET_ERROR); checkNotNull(hbaseTable, ConfigConstants.TABLE_NAME + NOT_SET_ERROR); checkNotNull(columnFamily, ConfigConstants.COLUMN_FAMILY + NOT_SET_ERROR); LOG.info("Zookeeper: " + conf.get(HConstants.ZOOKEEPER_QUORUM)); LOG.info("SQL Table: " + sqlTable); LOG.info("HBase Table: " + hbaseTable); LOG.info("HBase Column Family: " + columnFamily); LOG.info("Input separator: '" + separator + "'"); final HTablePool pool = new HTablePool(conf, 1); HBaseMetadata metadata = new HBaseMetadata(new PoolHTableProvider(hbaseTable, pool)); metadata.setColumnFamily(columnFamily); HBaseStore store = new HBaseStore(metadata, null, new MetadataCache(metadata)); tableId = store.getTableId(sqlTable); schema = store.getSchema(sqlTable);/*from www .j a va2 s .c om*/ mutationFactory = new MutationFactory(store); mutationFactory.setColumnFamily(columnFamily); rowParser = new RowParser(schema, columns, separator); checkSqlColumnsMatch(sqlTable); }