Example usage for org.apache.hadoop.fs FileSystem isDirectory

Introduction

In this page you can find the example usage for org.apache.hadoop.fs FileSystem isDirectory.

Prototype

@Deprecated
public boolean isDirectory(Path f) throws IOException 

Source Link

Document

True iff the named path is a directory.

Usage

From source file:org.apache.tajo.engine.function.hiveudf.HiveFunctionLoader.java

License:Apache License

public static Optional<List<FunctionDesc>> loadHiveUDFs(TajoConf conf) {
    ArrayList<FunctionDesc> funcList = new ArrayList<>();
    String udfdir = conf.getVar(TajoConf.ConfVars.HIVE_UDF_JAR_DIR);

    try {//from www  .j av  a  2 s  . c om
        // Currently Hive udf jar must be on local filesystem
        FileSystem fs = FileSystem.getLocal(conf);
        Path udfPath = new Path(udfdir);

        if (!fs.isDirectory(udfPath)) {
            LOG.warn("Hive UDF directory doesn't exist : " + udfdir);
            return Optional.empty();
        }

        // Read jar paths from the directory and change to URLs
        URL[] urls = Arrays.stream(fs.listStatus(udfPath, (Path path) -> path.getName().endsWith(".jar")))
                .map(fstatus -> {
                    try {
                        return new URL("jar:" + fstatus.getPath().toUri().toURL() + "!/");
                    } catch (MalformedURLException e) {
                        e.printStackTrace();
                    }

                    return null;
                }).toArray(URL[]::new);

        // Extract UDF classes and build function information
        Set<Class<? extends UDF>> udfClasses = getSubclassesFromJarEntry(urls, UDF.class);
        buildFunctionsFromUDF(udfClasses, funcList, "jar:" + urls[0].getPath());

    } catch (IOException e) {
        throw new TajoInternalError(e);
    }

    return Optional.of(funcList);
}

---

From source file:org.apache.tajo.engine.planner.physical.PhysicalPlanUtil.java

License:Apache License

/**
 *
 * @param fs/* ww w. j a va  2s.  co  m*/
 * @param path The table path
 * @param result The final result files to be used
 * @param startFileIndex
 * @param numResultFiles
 * @param currentFileIndex
 * @param partitioned A flag to indicate if this table is partitioned
 * @param currentDepth Current visiting depth of partition directories
 * @param maxDepth The partition depth of this table
 * @throws IOException
 */
private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result,
        int startFileIndex, int numResultFiles, AtomicInteger currentFileIndex, boolean partitioned,
        int currentDepth, int maxDepth) throws IOException {
    // Intermediate directory
    if (fs.isDirectory(path)) {
        FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter);
        if (files != null && files.length > 0) {

            for (FileStatus eachFile : files) {

                // checking if the enough number of files are found
                if (result.size() >= numResultFiles) {
                    return;
                }

                if (eachFile.isDirectory()) {
                    getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles,
                            currentFileIndex, partitioned, currentDepth + 1, // increment a visiting depth
                            maxDepth);

                    // if partitioned table, we should ignore files located in the intermediate directory.
                    // we can ensure that this file is in leaf directory if currentDepth == maxDepth.
                } else if (eachFile.isFile() && eachFile.getLen() > 0
                        && (!partitioned || currentDepth == maxDepth)) {
                    if (currentFileIndex.get() >= startFileIndex) {
                        result.add(eachFile);
                    }
                    currentFileIndex.incrementAndGet();
                }
            }
        }

        // Files located in leaf directory
    } else {
        FileStatus fileStatus = fs.getFileStatus(path);
        if (fileStatus != null && fileStatus.getLen() > 0) {
            if (currentFileIndex.get() >= startFileIndex) {
                result.add(fileStatus);
            }
            currentFileIndex.incrementAndGet();
            if (result.size() >= numResultFiles) {
                return;
            }
        }
    }
}

---

From source file:org.apache.tajo.engine.planner.PlannerUtil.java

License:Apache License

private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result,
        int startFileIndex, int numResultFiles, AtomicInteger currentFileIndex) throws IOException {
    if (fs.isDirectory(path)) {
        FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter);
        if (files != null && files.length > 0) {
            for (FileStatus eachFile : files) {
                if (result.size() >= numResultFiles) {
                    return;
                }/*from  w ww. j a  va  2s . c o m*/
                if (eachFile.isDirectory()) {
                    getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles,
                            currentFileIndex);
                } else if (eachFile.isFile() && eachFile.getLen() > 0) {
                    if (currentFileIndex.get() >= startFileIndex) {
                        result.add(eachFile);
                    }
                    currentFileIndex.incrementAndGet();
                }
            }
        }
    } else {
        FileStatus fileStatus = fs.getFileStatus(path);
        if (fileStatus != null && fileStatus.getLen() > 0) {
            if (currentFileIndex.get() >= startFileIndex) {
                result.add(fileStatus);
            }
            currentFileIndex.incrementAndGet();
            if (result.size() >= numResultFiles) {
                return;
            }
        }
    }
}

---

From source file:org.apache.tajo.engine.query.TestCTASQuery.java

License:Apache License

@Test
public final void testCtasWithoutTableDefinition() throws Exception {
    ResultSet res = executeQuery();
    res.close();/*from  w  ww. j  a  v  a2s.  c  om*/

    String tableName = CatalogUtil.normalizeIdentifier("testCtasWithoutTableDefinition");
    CatalogService catalog = testBase.getTestingCluster().getMaster().getCatalog();
    String qualifiedTableName = buildFQName(DEFAULT_DATABASE_NAME, tableName);
    TableDesc desc = catalog.getTableDesc(qualifiedTableName);
    assertTrue(catalog.existsTable(qualifiedTableName));

    assertTrue(desc.getSchema().contains("default.testctaswithouttabledefinition.col1"));
    PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
    assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
    assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());

    FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
    Path path = new Path(desc.getPath());
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    ResultSet res2 = executeFile("check1.sql");

    Map<Double, int[]> resultRows1 = Maps.newHashMap();
    resultRows1.put(45.0d, new int[] { 3, 2 });
    resultRows1.put(38.0d, new int[] { 2, 2 });

    int i = 0;
    while (res2.next()) {
        assertEquals(resultRows1.get(res2.getDouble(3))[0], res2.getInt(1));
        assertEquals(resultRows1.get(res2.getDouble(3))[1], res2.getInt(2));
        i++;
    }
    res2.close();
    assertEquals(2, i);
}

---

From source file:org.apache.tajo.engine.query.TestCTASQuery.java

License:Apache License

@Test
public final void testCtasWithColumnedPartition() throws Exception {
    ResultSet res = executeQuery();
    res.close();//from w w w  .j  av a2 s.  c o  m

    String tableName = CatalogUtil.normalizeIdentifier("testCtasWithColumnedPartition");

    TajoTestingCluster cluster = testBase.getTestingCluster();
    CatalogService catalog = cluster.getMaster().getCatalog();
    TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));
    PartitionMethodDesc partitionDesc = desc.getPartitionMethod();
    assertEquals(partitionDesc.getPartitionType(), CatalogProtos.PartitionType.COLUMN);
    assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());

    FileSystem fs = FileSystem.get(cluster.getConfiguration());
    Path path = new Path(desc.getPath());
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
    if (!cluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    ResultSet res2 = executeFile("check2.sql");

    Map<Double, int[]> resultRows1 = Maps.newHashMap();
    resultRows1.put(45.0d, new int[] { 3, 2 });
    resultRows1.put(38.0d, new int[] { 2, 2 });

    int i = 0;
    while (res2.next()) {
        assertEquals(resultRows1.get(res2.getDouble(3))[0], res2.getInt(1));
        assertEquals(resultRows1.get(res2.getDouble(3))[1], res2.getInt(2));
        i++;
    }
    res2.close();
    assertEquals(2, i);
}

---

From source file:org.apache.tajo.engine.query.TestTablePartitions.java

License:Apache License

private void assertPartitionDirectories(TableDesc desc) throws IOException {
    FileSystem fs = FileSystem.get(conf);
    Path path = new Path(desc.getPath());
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=49.0")));
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }/*from   w w w . j av  a2 s.  com*/
}

---

From source file:org.apache.tajo.engine.query.TestTablePartitions.java

License:Apache License

@Test
public final void testColumnPartitionedTableByThreeColumns() throws Exception {
    String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumns");
    ResultSet res = testBase.execute("create table " + tableName
            + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
    res.close();/* w  w w  .  j av  a 2  s  .  c  o  m*/
    TajoTestingCluster cluster = testBase.getTestingCluster();
    CatalogService catalog = cluster.getMaster().getCatalog();
    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));

    res = executeString("insert overwrite into " + tableName
            + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
    res.close();

    TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    Path path = new Path(desc.getPath());

    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0")));
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    res = executeString("select * from " + tableName + " where col2 = 2");

    Map<Double, int[]> resultRows1 = Maps.newHashMap();
    resultRows1.put(45.0d, new int[] { 3, 2 });
    resultRows1.put(38.0d, new int[] { 2, 2 });

    for (int i = 0; i < 2; i++) {
        assertTrue(res.next());
        assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(2));
        assertEquals(resultRows1.get(res.getDouble(4))[1], res.getInt(3));
    }
    res.close();

    Map<Double, int[]> resultRows2 = Maps.newHashMap();
    resultRows2.put(49.0d, new int[] { 3, 3 });
    resultRows2.put(45.0d, new int[] { 3, 2 });
    resultRows2.put(38.0d, new int[] { 2, 2 });

    res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2");

    for (int i = 0; i < 3; i++) {
        assertTrue(res.next());
        assertEquals(resultRows2.get(res.getDouble(4))[0], res.getInt(2));
        assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3));
    }
    res.close();
}

---

From source file:org.apache.tajo.engine.query.TestTablePartitions.java

License:Apache License

@Test
public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Exception {
    String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns");
    ResultSet res = testBase.execute("create table " + tableName
            + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) ");
    res.close();/*from   w w w. j  a  va 2s .  co  m*/
    TajoTestingCluster cluster = testBase.getTestingCluster();
    CatalogService catalog = cluster.getMaster().getCatalog();
    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));

    res = executeString("insert into " + tableName
            + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
    res.close();

    TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    Path path = new Path(desc.getPath());

    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0")));
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    res = executeString("select * from " + tableName + " where col2 = 2");

    Map<Double, int[]> resultRows1 = Maps.newHashMap();
    resultRows1.put(45.0d, new int[] { 3, 2 });
    resultRows1.put(38.0d, new int[] { 2, 2 });

    for (int i = 0; i < 2; i++) {
        assertTrue(res.next());
        assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(2));
        assertEquals(resultRows1.get(res.getDouble(4))[1], res.getInt(3));
    }
    res.close();

    Map<Double, int[]> resultRows2 = Maps.newHashMap();
    resultRows2.put(49.0d, new int[] { 3, 3 });
    resultRows2.put(45.0d, new int[] { 3, 2 });
    resultRows2.put(38.0d, new int[] { 2, 2 });

    res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2");

    for (int i = 0; i < 3; i++) {
        assertTrue(res.next());
        assertEquals(resultRows2.get(res.getDouble(4))[0], res.getInt(2));
        assertEquals(resultRows2.get(res.getDouble(4))[1], res.getInt(3));
    }
    res.close();

    // insert into already exists partitioned table
    res = executeString("insert into " + tableName
            + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem");
    res.close();

    desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    path = new Path(desc.getPath());

    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0")));

    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }
    String expected = "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "N\n" + "R\n" + "R\n" + "R\n" + "R\n";

    String tableData = getTableFileContents(new Path(desc.getPath()));
    assertEquals(expected, tableData);

    res = executeString("select * from " + tableName + " where col2 = 2");
    String resultSetData = resultSetToString(res);
    res.close();
    expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n"
            + "R,3,2,45.0\n" + "R,3,2,45.0\n";
    assertEquals(expected, resultSetData);

    res = executeString("select * from " + tableName + " where (col1 = 2 or col1 = 3) and col2 >= 2");
    resultSetData = resultSetToString(res);
    res.close();
    expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,2,2,38.0\n" + "N,2,2,38.0\n"
            + "R,3,2,45.0\n" + "R,3,2,45.0\n" + "R,3,3,49.0\n" + "R,3,3,49.0\n";
    assertEquals(expected, resultSetData);

    // Check not to remove existing partition directories.
    res = executeString("insert overwrite into " + tableName
            + " select l_returnflag, l_orderkey, l_partkey, 30.0 as l_quantity from lineitem "
            + " where l_orderkey = 1 and l_partkey = 1 and  l_linenumber = 1");
    res.close();

    desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    assertTrue(fs.isDirectory(path));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=30.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2/col3=38.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2/col3=45.0")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3/col3=49.0")));

    if (!testingCluster.isHCatalogStoreRunning()) {
        // TODO: If there is existing another partition directory, we must add its rows number to result row numbers.
        // assertEquals(6, desc.getStats().getNumRows().intValue());
    }

    res = executeString("select * from " + tableName + " where col2 = 1");
    resultSetData = resultSetToString(res);
    res.close();
    expected = "col4,col1,col2,col3\n" + "-------------------------------\n" + "N,1,1,17.0\n" + "N,1,1,17.0\n"
            + "N,1,1,30.0\n" + "N,1,1,36.0\n" + "N,1,1,36.0\n";

    assertEquals(expected, resultSetData);

    // insert overwrite empty result to partitioned table
    res = executeString("insert overwrite into " + tableName
            + " select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem where l_orderkey"
            + " > 100");
    res.close();

    desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);

    ContentSummary summary = fs.getContentSummary(new Path(desc.getPath()));

    assertEquals(summary.getDirectoryCount(), 1L);
    assertEquals(summary.getFileCount(), 0L);
    assertEquals(summary.getLength(), 0L);
}

---

From source file:org.apache.tajo.engine.query.TestTablePartitions.java

License:Apache License

@Test
public final void testColumnPartitionedTableByOneColumnsWithCompression() throws Exception {
    String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumnsWithCompression");
    ResultSet res = executeString("create table " + tableName + " (col2 int4, col3 float8) USING csv "
            + "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') "
            + "PARTITION BY column(col1 int4)");
    res.close();//from ww w .j a  va2 s  .  co m
    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));

    res = executeString(
            "insert overwrite into " + tableName + " select l_partkey, l_quantity, l_orderkey from lineitem");
    res.close();
    TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.exists(new Path(desc.getPath())));
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);

    Path path = new Path(desc.getPath());
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));

    for (FileStatus partition : fs.listStatus(path)) {
        assertTrue(fs.isDirectory(partition.getPath()));
        for (FileStatus file : fs.listStatus(partition.getPath())) {
            CompressionCodec codec = factory.getCodec(file.getPath());
            assertTrue(codec instanceof DeflateCodec);
        }
    }
}

---

From source file:org.apache.tajo.engine.query.TestTablePartitions.java

License:Apache License

@Test
public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws Exception {
    String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByTwoColumnsWithCompression");
    ResultSet res = executeString("create table " + tableName + " (col3 float8, col4 text) USING csv "
            + "WITH ('csvfile.delimiter'='|','compression.codec'='org.apache.hadoop.io.compress.DeflateCodec') "
            + "PARTITION by column(col1 int4, col2 int4)");
    res.close();/*from  w ww . j  a v a 2  s .  c o m*/

    assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName));

    res = executeString("insert overwrite into " + tableName
            + " select  l_quantity, l_returnflag, l_orderkey, l_partkey from lineitem");
    res.close();
    TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
    if (!testingCluster.isHCatalogStoreRunning()) {
        assertEquals(5, desc.getStats().getNumRows().intValue());
    }

    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.exists(new Path(desc.getPath())));
    CompressionCodecFactory factory = new CompressionCodecFactory(conf);

    Path path = new Path(desc.getPath());
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=2")));
    assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3/col2=3")));

    for (FileStatus partition1 : fs.listStatus(path)) {
        assertTrue(fs.isDirectory(partition1.getPath()));
        for (FileStatus partition2 : fs.listStatus(partition1.getPath())) {
            assertTrue(fs.isDirectory(partition2.getPath()));
            for (FileStatus file : fs.listStatus(partition2.getPath())) {
                CompressionCodec codec = factory.getCodec(file.getPath());
                assertTrue(codec instanceof DeflateCodec);
            }
        }
    }
}

---

• «
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• »