Example usage for org.apache.hadoop.fs FileSystem FS_DEFAULT_NAME_KEY

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem FS_DEFAULT_NAME_KEY.

Prototype

String FS_DEFAULT_NAME_KEY

To view the source code for org.apache.hadoop.fs FileSystem FS_DEFAULT_NAME_KEY.

Click Source Link

Usage

From source file:org.apache.drill.exec.physical.impl.writer.TestParquetWriterEmptyFiles.java

License:Apache License

@BeforeClass
public static void initFs() throws Exception {
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "local");

    fs = FileSystem.get(conf);/*w w  w  . ja v  a 2s. c om*/

    updateTestCluster(3, null);
}

---

From source file:org.apache.drill.exec.physical.impl.writer.TestWriter.java

License:Apache License

@BeforeClass
public static void initFs() throws Exception {
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "local");

    fs = FileSystem.get(conf);/*  www  . j  av a  2s  .  c om*/
}

---

From source file:org.apache.drill.exec.physical.unit.TestMiniPlan.java

License:Apache License

@BeforeClass
public static void initFS() throws Exception {
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
    fs = new DrillFileSystem(conf);
}

---

From source file:org.apache.drill.exec.store.dfs.FileSystemPlugin.java

License:Apache License

public FileSystemPlugin(FileSystemConfig config, DrillbitContext context, String name)
        throws ExecutionSetupException {
    this.config = config;
    this.lpPersistance = context.getLpPersistence();
    try {// ww w  . j  a v a 2 s. c  o  m

        fsConf = new Configuration();
        if (config.config != null) {
            for (String s : config.config.keySet()) {
                fsConf.set(s, config.config.get(s));
            }
        }
        fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, config.connection);
        fsConf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
        fsConf.set("fs.drill-local.impl", LocalSyncableFileSystem.class.getName());

        formatCreator = newFormatCreator(config, context, fsConf);
        List<FormatMatcher> matchers = Lists.newArrayList();
        formatPluginsByConfig = Maps.newHashMap();
        for (FormatPlugin p : formatCreator.getConfiguredFormatPlugins()) {
            matchers.add(p.getMatcher());
            formatPluginsByConfig.put(p.getConfig(), p);
        }

        final boolean noWorkspace = config.workspaces == null || config.workspaces.isEmpty();
        List<WorkspaceSchemaFactory> factories = Lists.newArrayList();
        if (!noWorkspace) {
            for (Map.Entry<String, WorkspaceConfig> space : config.workspaces.entrySet()) {
                factories.add(new WorkspaceSchemaFactory(this, space.getKey(), name, space.getValue(), matchers,
                        context.getLpPersistence(), context.getClasspathScan()));
            }
        }

        // if the "default" workspace is not given add one.
        if (noWorkspace || !config.workspaces.containsKey(DEFAULT_WS_NAME)) {
            factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT,
                    matchers, context.getLpPersistence(), context.getClasspathScan()));
        }

        this.schemaFactory = new FileSystemSchemaFactory(name, factories);
    } catch (IOException e) {
        throw new ExecutionSetupException("Failure setting up file system plugin.", e);
    }
}

---

From source file:org.apache.drill.exec.store.dfs.TestDrillFileSystem.java

License:Apache License

@Test
public void testIOStats() throws Exception {
    DrillFileSystem dfs = null;/* w w w  . j av a 2  s.com*/
    InputStream is = null;
    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
    OpProfileDef profileDef = new OpProfileDef(0 /*operatorId*/, 0 /*operatorType*/, 0 /*inputCount*/);
    OperatorStats stats = new OperatorStats(profileDef, null /*allocator*/);

    // start wait time method in OperatorStats expects the OperatorStats state to be in "processing"
    stats.startProcessing();

    try {
        dfs = new DrillFileSystem(conf, stats);
        is = dfs.open(new Path(tempFilePath));

        byte[] buf = new byte[8000];
        while (is.read(buf, 0, buf.length) != -1) {
        }
    } finally {
        stats.stopProcessing();

        if (is != null) {
            is.close();
        }

        if (dfs != null) {
            dfs.close();
        }
    }

    OperatorProfile operatorProfile = stats.getProfile();
    assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0);
}

---

From source file:org.apache.drill.exec.store.easy.json.JSONFormatPlugin.java

License:Apache License

@Override
public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
    Map<String, String> options = Maps.newHashMap();

    options.put("location", writer.getLocation());

    FragmentHandle handle = context.getHandle();
    String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
    options.put("prefix", fragmentId);

    options.put("separator", " ");
    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).connection);

    options.put("extension", "json");
    options.put("extended",
            Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_EXTENDED_TYPES)));
    options.put("uglify", Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_UGLIFY)));
    options.put("skipnulls",
            Boolean.toString(context.getOptions().getOption(ExecConstants.JSON_WRITER_SKIPNULLFIELDS)));

    RecordWriter recordWriter = new JsonRecordWriter();
    recordWriter.init(options);/*w  w  w .j  a  va  2 s  . co m*/

    return recordWriter;
}

---

From source file:org.apache.drill.exec.store.easy.json.JsonRecordWriter.java

License:Apache License

@Override
public void init(Map<String, String> writerOptions) throws IOException {
    this.location = writerOptions.get("location");
    this.prefix = writerOptions.get("prefix");
    this.fieldDelimiter = writerOptions.get("separator");
    this.extension = writerOptions.get("extension");
    this.useExtendedOutput = Boolean.parseBoolean(writerOptions.get("extended"));
    this.skipNullFields = Boolean.parseBoolean(writerOptions.get("skipnulls"));
    final boolean uglify = Boolean.parseBoolean(writerOptions.get("uglify"));

    Configuration conf = new Configuration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, writerOptions.get(FileSystem.FS_DEFAULT_NAME_KEY));
    this.fs = FileSystem.get(conf);

    Path fileName = new Path(location, prefix + "_" + index + "." + extension);
    try {/*from   ww w.  j  a v a  2s.  c o  m*/
        stream = fs.create(fileName);
        JsonGenerator generator = factory.createGenerator(stream).useDefaultPrettyPrinter();
        if (uglify) {
            generator = generator.setPrettyPrinter(new MinimalPrettyPrinter(LINE_FEED));
        }
        if (useExtendedOutput) {
            gen = new ExtendedJsonOutput(generator);
        } else {
            gen = new BasicJsonOutput(generator);
        }
        logger.debug("Created file: {}", fileName);
    } catch (IOException ex) {
        logger.error("Unable to create file: " + fileName, ex);
        throw ex;
    }
}

---

From source file:org.apache.drill.exec.store.easy.model.ModelFormatPlugin.java

License:Apache License

@Override
public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
    Map<String, String> options = Maps.newHashMap();

    options.put("location", writer.getLocation());

    FragmentHandle handle = context.getHandle();
    String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
    options.put("prefix", fragmentId);

    options.put("separator", ((ModelFormatConfig) getConfig()).getDelimiter());
    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).connection);

    options.put("extension", ((ModelFormatConfig) getConfig()).getExtensions().get(0));

    RecordWriter recordWriter = new DrillModelWriter(/*context.getAllocator()*/);
    recordWriter.init(options);/*  w w  w  .j  a  va  2  s . co m*/

    return recordWriter;
}

---

From source file:org.apache.drill.exec.store.easy.text.TextFormatPlugin.java

License:Apache License

@Override
public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer) throws IOException {
    final Map<String, String> options = Maps.newHashMap();

    options.put("location", writer.getLocation());

    FragmentHandle handle = context.getHandle();
    String fragmentId = String.format("%d_%d", handle.getMajorFragmentId(), handle.getMinorFragmentId());
    options.put("prefix", fragmentId);

    options.put("separator", ((TextFormatConfig) getConfig()).getFieldDelimiterAsString());
    options.put(FileSystem.FS_DEFAULT_NAME_KEY, ((FileSystemConfig) writer.getStorageConfig()).connection);

    options.put("extension", ((TextFormatConfig) getConfig()).getExtensions().get(0));

    RecordWriter recordWriter = new DrillTextRecordWriter(context.getAllocator());
    recordWriter.init(options);//from w w w .  j  a v  a  2s. c  o m

    return recordWriter;
}

---

From source file:org.apache.drill.exec.store.hive.HiveTestDataGenerator.java

License:Apache License

private HiveTestDataGenerator(final String dbDir, final String whDir) {
    this.dbDir = dbDir;
    this.whDir = whDir;

    config = Maps.newHashMap();/*w  w  w .  ja va  2 s.  c om*/
    config.put("hive.metastore.uris", "");
    config.put("javax.jdo.option.ConnectionURL",
            String.format("jdbc:derby:;databaseName=%s;create=true", dbDir));
    config.put("hive.metastore.warehouse.dir", whDir);
    config.put(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• »