gobblin.data.management.copy.recovery.RecoveryHelperTest.java Source code

Introduction

Here is the source code for gobblin.data.management.copy.recovery.RecoveryHelperTest.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.data.management.copy.recovery;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import com.google.common.base.Optional;
import com.google.common.base.Predicates;
import com.google.common.io.Files;

import gobblin.configuration.ConfigurationKeys;
import gobblin.configuration.State;
import gobblin.data.management.copy.CopyConfiguration;
import gobblin.data.management.copy.CopySource;
import gobblin.data.management.copy.CopyableFile;
import gobblin.data.management.copy.PreserveAttributes;
import gobblin.util.guid.Guid;

public class RecoveryHelperTest {

    private File tmpDir;

    @BeforeMethod
    public void setUp() throws Exception {
        this.tmpDir = Files.createTempDir();
        this.tmpDir.deleteOnExit();
    }

    @Test
    public void testGetPersistDir() throws Exception {

        State state = new State();

        Assert.assertFalse(RecoveryHelper.getPersistDir(state).isPresent());

        state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath());

        Assert.assertTrue(RecoveryHelper.getPersistDir(state).isPresent());
        Assert.assertTrue(RecoveryHelper.getPersistDir(state).get().toUri().getPath()
                .startsWith(this.tmpDir.getAbsolutePath()));

    }

    @Test
    public void testPersistFile() throws Exception {

        String content = "contents";

        File stagingDir = Files.createTempDir();
        stagingDir.deleteOnExit();

        File file = new File(stagingDir, "file");
        OutputStream os = new FileOutputStream(file);
        IOUtils.write(content, os);
        os.close();

        Assert.assertEquals(stagingDir.listFiles().length, 1);

        State state = new State();
        state.setProp(RecoveryHelper.PERSIST_DIR_KEY, this.tmpDir.getAbsolutePath());
        state.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, "/publisher");

        File recoveryDir = new File(RecoveryHelper.getPersistDir(state).get().toUri().getPath());

        FileSystem fs = FileSystem.getLocal(new Configuration());

        CopyableFile copyableFile = CopyableFile.builder(fs, new FileStatus(0, false, 0, 0, 0, new Path("/file")),
                new Path("/dataset"), CopyConfiguration.builder(fs, state.getProperties())
                        .preserve(PreserveAttributes.fromMnemonicString("")).build())
                .build();

        CopySource.setWorkUnitGuid(state, Guid.fromHasGuid(copyableFile));

        RecoveryHelper recoveryHelper = new RecoveryHelper(FileSystem.getLocal(new Configuration()), state);

        recoveryHelper.persistFile(state, copyableFile, new Path(file.getAbsolutePath()));

        Assert.assertEquals(stagingDir.listFiles().length, 0);
        Assert.assertEquals(recoveryDir.listFiles().length, 1);

        File fileInRecovery = recoveryDir.listFiles()[0];
        Assert.assertEquals(IOUtils.readLines(new FileInputStream(fileInRecovery)).get(0), content);

        Optional<FileStatus> fileToRecover = recoveryHelper.findPersistedFile(state, copyableFile,
                Predicates.<FileStatus>alwaysTrue());
        Assert.assertTrue(fileToRecover.isPresent());
        Assert.assertEquals(fileToRecover.get().getPath().toUri().getPath(), fileInRecovery.getAbsolutePath());

        fileToRecover = recoveryHelper.findPersistedFile(state, copyableFile, Predicates.<FileStatus>alwaysFalse());
        Assert.assertFalse(fileToRecover.isPresent());

    }

    @Test
    public void testPurge() throws Exception {
        String content = "contents";

        File persistDirBase = Files.createTempDir();
        persistDirBase.deleteOnExit();

        State state = new State();
        state.setProp(RecoveryHelper.PERSIST_DIR_KEY, persistDirBase.getAbsolutePath());
        state.setProp(RecoveryHelper.PERSIST_RETENTION_KEY, "1");

        RecoveryHelper recoveryHelper = new RecoveryHelper(FileSystem.getLocal(new Configuration()), state);
        File persistDir = new File(RecoveryHelper.getPersistDir(state).get().toString());
        persistDir.mkdir();

        File file = new File(persistDir, "file1");
        OutputStream os = new FileOutputStream(file);
        IOUtils.write(content, os);
        os.close();
        file.setLastModified(System.currentTimeMillis() - TimeUnit.HOURS.toMillis(2));

        File file2 = new File(persistDir, "file2");
        OutputStream os2 = new FileOutputStream(file2);
        IOUtils.write(content, os2);
        os2.close();

        Assert.assertEquals(persistDir.listFiles().length, 2);

        recoveryHelper.purgeOldPersistedFile();

        Assert.assertEquals(persistDir.listFiles().length, 1);
    }

    @Test
    public void testShortenPathName() throws Exception {

        Assert.assertEquals(RecoveryHelper.shortenPathName(new Path("/test"), 10), "_test");
        Assert.assertEquals(RecoveryHelper.shortenPathName(new Path("/relatively/long/path"), 9), "_re...ath");

    }
}