List of usage examples for org.apache.hadoop.conf Configuration setClass
public void setClass(String name, Class<?> theClass, Class<?> xface)
name property to the name of a theClass implementing the given interface xface. From source file:org.apache.lens.cube.parse.TestCubeRewriter.java
License:Apache License
@Test public void testMaxCoveringFact() throws Exception { Configuration conf = getConf(); conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); conf.set(DRIVER_SUPPORTED_STORAGES, "C1,C2,C4"); CubeQueryContext cubeQueryContext = rewriteCtx("select SUM(msr2) from testCube where " + THIS_YEAR_RANGE, conf);/*w w w . ja va 2s.c o m*/ PruneCauses<CubeFactTable> pruneCause = cubeQueryContext.getFactPruningMsgs(); int lessDataCauses = 0; for (Map.Entry<CubeFactTable, List<CandidateTablePruneCause>> entry : pruneCause.entrySet()) { for (CandidateTablePruneCause cause : entry.getValue()) { if (cause.getCause().equals(LESS_DATA)) { lessDataCauses++; } } } assertTrue(lessDataCauses > 0); }
From source file:org.apache.lens.cube.parse.TestCubeRewriter.java
License:Apache License
@Test public void testConvertDimFilterToFactFilterForSingleFact() throws Exception { Configuration conf = getConf(); conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, false); conf.set(DRIVER_SUPPORTED_STORAGES, "C3"); conf.setBoolean(DISABLE_AUTO_JOINS, false); conf.setBoolean(REWRITE_DIM_FILTER_TO_FACT_FILTER, true); // filter with = String hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region = 'asia' and " + TWO_DAYS_RANGE, conf); String filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where ((cubecountry.region) = 'asia') and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); // filter with or hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where (cubecountry.region = 'asia' " + "or cubecountry.region = 'europe') and " + TWO_DAYS_RANGE, conf); filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where (((cubecountry.region) = 'asia') or ((cubecountry.region) = 'europe')) " + "and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); //filter with in hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region in ('asia','europe') " + "and " + TWO_DAYS_RANGE, conf); filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where (cubecountry.region) in ('asia' , 'europe') and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); //filter with not in hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region not in ('asia','europe') " + "and " + TWO_DAYS_RANGE, conf); filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where (cubecountry.region) not in ('asia' , 'europe') and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); //filter with != hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region != 'asia' " + "and " + TWO_DAYS_RANGE, conf); filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where ((cubecountry.region) != 'asia') and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); //filter with cube alias hql = rewrite("select cubecountry.name, msr2 from" + " testCube as t" + " where cubecountry.region = 'asia' " + "and zipcode = 'x' and " + TWO_DAYS_RANGE, conf); filterSubquery = "t.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where ((cubecountry.region) = 'asia') and (cubecountry.dt = 'latest') )"; assertTrue(hql.contains(filterSubquery)); //filter with AbridgedTimeRangeWriter conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); hql = rewrite("select cubecountry.name, msr2 from" + " testCube" + " where cubecountry.region = 'asia' and " + TWO_DAYS_RANGE, conf);/*ww w .ja v a 2 s .c o m*/ filterSubquery = "testcube.countryid in ( select id from TestQueryRewrite.c3_countrytable_partitioned " + "cubecountry where ((cubecountry.region) = 'asia') and (cubecountry.dt = 'latest') )"; String timeKeyIn = "(testcube.dt) in"; assertTrue(hql.contains(timeKeyIn)); assertTrue(hql.contains(filterSubquery)); }
From source file:org.apache.lens.cube.parse.TestCubeRewriter.java
License:Apache License
@Test public void testLookAhead() throws Exception { Configuration conf = getConf(); conf.set(CubeQueryConfUtil.PROCESS_TIME_PART_COL, "pt"); conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, AbridgedTimeRangeWriter.class, TimeRangeWriter.class); CubeQueryContext ctx = rewriteCtx(/*from ww w. j a v a2 s. c o m*/ "select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); assertEquals(ctx.candidateFacts.size(), 1); CandidateFact candidateFact = ctx.candidateFacts.iterator().next(); Set<FactPartition> partsQueried = new TreeSet<>(candidateFact.getPartsQueried()); Date ceilDay = DAILY.getCeilDate(getDateWithOffset(DAILY, -2)); Date nextDay = DateUtils.addDays(ceilDay, 1); Date nextToNextDay = DateUtils.addDays(nextDay, 1); HashSet<String> storageTables = Sets.newHashSet(); for (String storageTable : candidateFact.getStorageTables()) { storageTables.add(storageTable.split("\\.")[1]); } TreeSet<FactPartition> expectedPartsQueried = Sets.newTreeSet(); for (TimePartition p : Iterables.concat( TimePartition.of(HOURLY, getDateWithOffset(DAILY, -2)).rangeUpto(TimePartition.of(HOURLY, ceilDay)), TimePartition.of(DAILY, ceilDay).rangeUpto(TimePartition.of(DAILY, nextDay)), TimePartition.of(HOURLY, nextDay).rangeUpto(TimePartition.of(HOURLY, NOW)))) { FactPartition fp = new FactPartition("it", p, null, storageTables); expectedPartsQueried.add(fp); } for (TimePartition it : TimePartition.of(HOURLY, ceilDay).rangeUpto(TimePartition.of(HOURLY, nextDay))) { for (TimePartition pt : TimePartition.of(HOURLY, nextDay) .rangeUpto(TimePartition.of(HOURLY, nextToNextDay))) { FactPartition ptPartition = new FactPartition("pt", pt, null, storageTables); FactPartition itPartition = new FactPartition("it", it, ptPartition, storageTables); expectedPartsQueried.add(itPartition); } } assertEquals(partsQueried, expectedPartsQueried); conf.setInt(CubeQueryConfUtil.LOOK_AHEAD_PT_PARTS_PFX, 3); ctx = rewriteCtx("select dim1, max(msr3)," + " msr2 from testCube" + " where " + TWO_DAYS_RANGE_IT, conf); partsQueried = new TreeSet<>(ctx.candidateFacts.iterator().next().getPartsQueried()); // pt does not exist beyond 1 day. So in this test, max look ahead possible is 3 assertEquals(partsQueried, expectedPartsQueried); }
From source file:org.apache.mahout.avro.text.mapreduce.AvroDocumentProcessor.java
License:Apache License
@Override public int run(String[] args) throws Exception { Configuration conf = new Configuration(); if (args.length != 2) { System.err.println("Usage: wordcount <in> <out>"); return 0; }/*from www . ja v a 2 s . c om*/ conf.setStrings("io.serializations", new String[] { WritableSerialization.class.getName(), AvroSpecificSerialization.class.getName(), AvroReflectSerialization.class.getName(), AvroGenericSerialization.class.getName() }); AvroComparator.setSchema(AvroDocument._SCHEMA); //TODO: must be done in mapper, reducer configure method. conf.setClass("mapred.output.key.comparator.class", AvroComparator.class, RawComparator.class); Job job = new Job(conf, "document processor"); job.setJarByClass(AvroDocumentProcessor.class); job.setMapperClass(TokenizerMapper.class); job.setReducerClass(Reducer.class); job.setOutputKeyClass(AvroDocument.class); job.setOutputValueClass(NullWritable.class); job.setInputFormatClass(AvroInputFormat.class); job.setOutputFormatClass(AvroOutputFormat.class); Path input = new Path(args[0]); Path output = new Path(args[1]); FileSystem fs = FileSystem.get(conf); fs.delete(output, true); AvroInputFormat.setAvroInputClass(job, AvroDocument.class); AvroOutputFormat.setAvroOutputClass(job, AvroDocument.class); FileInputFormat.addInputPath(job, input); FileOutputFormat.setOutputPath(job, output); return job.waitForCompletion(true) ? 0 : 1; }
From source file:org.apache.mahout.cf.taste.hadoop.RecommenderJob.java
License:Apache License
public static Configuration buildJobConf(String recommendClassName, int recommendationsPerUser, String userIDFile, String dataModelFile, String outputPath) throws IOException { Configuration jobConf = new Configuration(); FileSystem fs = FileSystem.get(jobConf); Path userIDFilePath = new Path(userIDFile).makeQualified(fs); Path outputPathPath = new Path(outputPath).makeQualified(fs); if (fs.exists(outputPathPath)) { fs.delete(outputPathPath, true); }/*from w w w. ja va 2 s. com*/ jobConf.set(RecommenderMapper.RECOMMENDER_CLASS_NAME, recommendClassName); jobConf.set(RecommenderMapper.RECOMMENDATIONS_PER_USER, String.valueOf(recommendationsPerUser)); jobConf.set(RecommenderMapper.DATA_MODEL_FILE, dataModelFile); jobConf.setClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class); jobConf.set("mapred.input.dir", StringUtils.escapeString(userIDFilePath.toString())); jobConf.setClass("mapred.mapper.class", RecommenderMapper.class, Mapper.class); jobConf.setClass("mapred.mapoutput.key.class", LongWritable.class, Object.class); jobConf.setClass("mapred.mapoutput.value.class", RecommendedItemsWritable.class, Object.class); jobConf.setClass("mapred.reducer.class", IdentityReducer.class, Reducer.class); jobConf.setClass("mapred.output.key.class", LongWritable.class, Object.class); jobConf.setClass("mapred.output.value.class", RecommendedItemsWritable.class, Object.class); jobConf.setClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class); jobConf.set("mapred.output.dir", StringUtils.escapeString(outputPathPath.toString())); return jobConf; }
From source file:org.apache.mahout.cf.taste.hadoop.SlopeOneDiffsToAveragesJob.java
License:Apache License
public static Configuration buildJobConf(String prefsFile, String outputPath) throws IOException { Configuration jobConf = new Configuration(); FileSystem fs = FileSystem.get(jobConf); Path prefsFilePath = new Path(prefsFile).makeQualified(fs); Path outputPathPath = new Path(outputPath).makeQualified(fs); if (fs.exists(outputPathPath)) { fs.delete(outputPathPath, true); }// w w w.j a va 2s .c o m jobConf.setClass("mapred.input.format.class", SequenceFileInputFormat.class, InputFormat.class); jobConf.set("mapred.input.dir", StringUtils.escapeString(prefsFilePath.toString())); jobConf.setClass("mapred.mapper.class", Mapper.class, Mapper.class); jobConf.setClass("mapred.mapoutput.key.class", ItemItemWritable.class, Object.class); jobConf.setClass("mapred.mapoutput.value.class", FloatWritable.class, Object.class); jobConf.setClass("mapred.reducer.class", SlopeOneDiffsToAveragesReducer.class, Reducer.class); jobConf.setClass("mapred.output.key.class", ItemItemWritable.class, Object.class); jobConf.setClass("mapred.output.value.class", FloatWritable.class, Object.class); jobConf.setClass("mapred.output.format.class", TextOutputFormat.class, OutputFormat.class); jobConf.set("mapred.output.dir", StringUtils.escapeString(outputPathPath.toString())); return jobConf; }
From source file:org.apache.mahout.cf.taste.hadoop.SlopeOnePrefsToDiffsJob.java
License:Apache License
public static Configuration buildJobConf(String prefsFile, String outputPath) throws IOException { Configuration jobConf = new Configuration(); FileSystem fs = FileSystem.get(jobConf); Path prefsFilePath = new Path(prefsFile).makeQualified(fs); Path outputPathPath = new Path(outputPath).makeQualified(fs); if (fs.exists(outputPathPath)) { fs.delete(outputPathPath, true); }//from ww w. j a v a 2 s . c om jobConf.setClass("mapred.input.format.class", TextInputFormat.class, InputFormat.class); jobConf.set("mapred.input.dir", StringUtils.escapeString(prefsFilePath.toString())); jobConf.setClass("mapred.mapper.class", SlopeOnePrefsToDiffsMapper.class, Mapper.class); jobConf.setClass("mapred.mapoutput.key.class", Text.class, Object.class); jobConf.setClass("mapred.mapoutput.value.class", ItemPrefWritable.class, Object.class); jobConf.setClass("mapred.reducer.class", SlopeOnePrefsToDiffsReducer.class, Reducer.class); jobConf.setClass("mapred.output.key.class", ItemItemWritable.class, Object.class); jobConf.setClass("mapred.output.value.class", FloatWritable.class, Object.class); jobConf.setClass("mapred.output.format.class", SequenceFileOutputFormat.class, OutputFormat.class); jobConf.set("mapred.output.dir", StringUtils.escapeString(outputPathPath.toString())); return jobConf; }
From source file:org.apache.oozie.service.TestEventHandlerService.java
License:Apache License
@Before protected void setUp() throws Exception { super.setUp(); Services services = new Services(); Configuration conf = services.getConf(); conf.set(Services.CONF_SERVICE_EXT_CLASSES, "org.apache.oozie.service.EventHandlerService"); conf.setClass(EventHandlerService.CONF_LISTENERS, DummyJobEventListener.class, JobEventListener.class); services.init();/*from ww w .jav a 2 s .c om*/ output.setLength(0); }
From source file:org.apache.oozie.service.TestHASLAService.java
License:Apache License
protected void setUp() throws Exception { Configuration conf = new Configuration(false); conf.set(Services.CONF_SERVICE_EXT_CLASSES, "org.apache.oozie.service.EventHandlerService," + "org.apache.oozie.sla.service.SLAService"); conf.setClass(EventHandlerService.CONF_LISTENERS, DummySLAEventListener.class, SLAEventListener.class); conf.setLong(SLAService.CONF_JOB_EVENT_LATENCY, 0); // manually do check in this test conf.setInt(SLAService.CONF_SLA_CHECK_INITIAL_DELAY, 100000); conf.setInt(SLAService.CONF_SLA_CHECK_INTERVAL, 100000); conf.setInt(EventHandlerService.CONF_WORKER_THREADS, 0); super.setUp(conf); Services.get().setService(ZKJobsConcurrencyService.class); }
From source file:org.apache.oozie.sla.TestSLAEventGeneration.java
License:Apache License
@Override @Before/*w ww . j a v a 2 s. c o m*/ protected void setUp() throws Exception { super.setUp(); services = new Services(); Configuration conf = services.getConf(); setClassesToBeExcluded(conf, excludeServices); conf.set(Services.CONF_SERVICE_EXT_CLASSES, EventHandlerService.class.getName() + "," + SLAService.class.getName()); conf.setClass(EventHandlerService.CONF_LISTENERS, SLAJobEventListener.class, JobEventListener.class); conf.setInt(EventHandlerService.CONF_WORKER_INTERVAL, 10000); conf.setInt(EventHandlerService.CONF_WORKER_THREADS, 0); conf.setInt(EventHandlerService.CONF_BATCH_SIZE, 1); conf.setInt(OozieClient.SLA_DISABLE_ALERT_OLDER_THAN, -1); services.init(); jpa = services.get(JPAService.class); ehs = services.get(EventHandlerService.class); cal = Calendar.getInstance(); }