gobblin.data.management.copy.extractor.InputStreamExtractorTest.java Source code

Introduction

Here is the source code for gobblin.data.management.copy.extractor.InputStreamExtractorTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package gobblin.data.management.copy.extractor;

import gobblin.configuration.ConfigurationKeys;
import gobblin.data.management.copy.CopyConfiguration;
import gobblin.data.management.copy.CopyContext;
import gobblin.data.management.copy.CopyableFile;
import gobblin.data.management.copy.FileAwareInputStream;
import gobblin.data.management.copy.PreserveAttributes;

import java.io.IOException;
import java.util.Properties;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.testng.Assert;
import org.testng.annotations.Test;

public class InputStreamExtractorTest {

    @Test
    public void testReadRecord() throws Exception {
        CopyableFile file = getTestCopyableFile("inputStreamExtractorTest/first.txt");

        FileAwareInputStreamExtractor extractor = new FileAwareInputStreamExtractor(
                FileSystem.getLocal(new Configuration()), file);

        FileAwareInputStream fileAwareInputStream = extractor.readRecord(null);

        Assert.assertEquals(fileAwareInputStream.getFile().getOrigin().getPath(), file.getOrigin().getPath());
        Assert.assertEquals(IOUtils.toString(fileAwareInputStream.getInputStream()), "first");

        Assert.assertNull(extractor.readRecord(null));
    }

    private CopyableFile getTestCopyableFile(String resourcePath) throws IOException {
        String filePath = getClass().getClassLoader().getResource(resourcePath).getFile();
        FileStatus status = new FileStatus(0l, false, 0, 0l, 0l, new Path(filePath));

        Properties properties = new Properties();
        properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");

        return CopyableFile.fromOriginAndDestination(FileSystem.getLocal(new Configuration()), status,
                new Path("/destination"),
                CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties)
                        .preserve(PreserveAttributes.fromMnemonicString("")).build())
                .build();
    }
}