com.liveramp.hank.storage.curly.AbstractCurlyPartitionUpdater.java Source code

Introduction

Here is the source code for com.liveramp.hank.storage.curly.AbstractCurlyPartitionUpdater.java

Source

/**
 *  Copyright 2011 LiveRamp
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */

package com.liveramp.hank.storage.curly;

import com.liveramp.hank.coordinator.Domain;
import com.liveramp.hank.coordinator.DomainVersion;
import com.liveramp.hank.partition_server.PartitionUpdateTaskStatistics;
import com.liveramp.hank.storage.PartitionRemoteFileOps;
import com.liveramp.hank.storage.cueball.Cueball;
import com.liveramp.hank.storage.cueball.CueballFilePath;
import com.liveramp.hank.storage.cueball.ValueTransformer;
import com.liveramp.hank.storage.incremental.IncrementalPartitionUpdater;
import com.liveramp.hank.storage.incremental.IncrementalUpdatePlan;
import com.liveramp.hank.util.EncodingHelper;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedSet;

public abstract class AbstractCurlyPartitionUpdater extends IncrementalPartitionUpdater {

    private static final Logger LOG = LoggerFactory.getLogger(CurlyFastPartitionUpdater.class);

    protected final PartitionRemoteFileOps partitionRemoteFileOps;

    public AbstractCurlyPartitionUpdater(Domain domain, PartitionRemoteFileOps partitionRemoteFileOps,
            String localPartitionRoot) throws IOException {
        super(domain, localPartitionRoot, new CurlyUpdatePlanner(domain));
        this.partitionRemoteFileOps = partitionRemoteFileOps;
    }

    public static final class OffsetTransformer implements ValueTransformer {
        private final long[] offsetAdjustments;
        private final int offsetNumBytes;

        public OffsetTransformer(int offsetNumBytes, long[] offsetAdjustments) {
            this.offsetNumBytes = offsetNumBytes;
            this.offsetAdjustments = offsetAdjustments;
        }

        @Override
        public void transform(byte[] buf, int valueOff, int relIndex) {
            long adjustment = offsetAdjustments[relIndex];
            if (adjustment != 0) {
                long offset = EncodingHelper.decodeLittleEndianFixedWidthLong(buf, valueOff, offsetNumBytes);
                offset += adjustment;
                EncodingHelper.encodeLittleEndianFixedWidthLong(offset, buf, valueOff, offsetNumBytes);
            }
        }
    }

    @Override
    protected Integer detectCurrentVersionNumber() throws IOException {
        SortedSet<CueballFilePath> localCueballBases = Cueball.getBases(localPartitionRoot);
        SortedSet<CurlyFilePath> localCurlyBases = Curly.getBases(localPartitionRoot);
        if (localCueballBases.size() > 0 && localCurlyBases.size() > 0) {
            if (localCueballBases.last().getVersion() == localCurlyBases.last().getVersion()) {
                return localCurlyBases.last().getVersion();
            } else {
                return null;
            }
        } else {
            return null;
        }
    }

    @Override
    protected Set<DomainVersion> detectCachedBasesCore() throws IOException {
        return detectCachedVersions(Cueball.getBases(localPartitionRootCache),
                Curly.getBases(localPartitionRootCache));
    }

    @Override
    protected Set<DomainVersion> detectCachedDeltasCore() throws IOException {
        return detectCachedVersions(Cueball.getDeltas(localPartitionRootCache),
                Curly.getDeltas(localPartitionRootCache));
    }

    private Set<DomainVersion> detectCachedVersions(SortedSet<CueballFilePath> cueballCachedFiles,
            SortedSet<CurlyFilePath> curlyCachedFiles) throws IOException {
        // Record in a set all cached Cueball versions
        HashSet<Integer> cachedCueballVersions = new HashSet<Integer>();
        for (CueballFilePath cueballCachedFile : cueballCachedFiles) {
            cachedCueballVersions.add(cueballCachedFile.getVersion());
        }
        // Compute cached Curly versions
        Set<DomainVersion> cachedVersions = new HashSet<DomainVersion>();
        for (CurlyFilePath curlyCachedFile : curlyCachedFiles) {
            // Check that the corresponding Cueball version is also cached
            if (cachedCueballVersions.contains(curlyCachedFile.getVersion())) {
                DomainVersion version = domain.getVersion(curlyCachedFile.getVersion());
                if (version != null) {
                    cachedVersions.add(version);
                }
            }
        }
        return cachedVersions;
    }

    @Override
    protected void cleanCachedVersions() throws IOException {
        // Delete all cached versions
        FileUtils.deleteDirectory(new File(localPartitionRootCache));
    }

    protected abstract boolean shouldFetchCurlyVersion(DomainVersion version) throws IOException;

    @Override
    protected void fetchVersion(DomainVersion version, String fetchRoot) throws IOException {
        // Fetch Cueball version.
        fetchCueballVersion(version, fetchRoot);
        if (shouldFetchCurlyVersion(version)) {
            fetchCurlyVersion(version, fetchRoot);
        }
    }

    private void fetchCueballVersion(DomainVersion version, String fetchRoot) throws IOException {
        String cueballFileToFetch = Cueball.getName(version);
        LOG.info(
                "Fetching from " + partitionRemoteFileOps + " for file " + cueballFileToFetch + " to " + fetchRoot);
        partitionRemoteFileOps.copyToLocalRoot(cueballFileToFetch, fetchRoot);
    }

    private void fetchCurlyVersion(DomainVersion version, String fetchRoot) throws IOException {
        String curlyFileToFetch = Curly.getName(version);
        LOG.info("Fetching from " + partitionRemoteFileOps + " for file " + curlyFileToFetch + " to " + fetchRoot);
        partitionRemoteFileOps.copyToLocalRoot(curlyFileToFetch, fetchRoot);
    }

    @Override
    protected abstract void runUpdateCore(DomainVersion currentVersion, DomainVersion updatingToVersion,
            IncrementalUpdatePlan updatePlan, String updateWorkRoot, PartitionUpdateTaskStatistics statistics)
            throws IOException;

    public CurlyFilePath getCurlyFilePathForVersion(DomainVersion version, DomainVersion currentVersion,
            boolean isBase) {
        if (currentVersion != null && currentVersion.equals(version)) {
            // If version is current version, data is in root
            return new CurlyFilePath(localPartitionRoot + "/" + Curly.getName(version.getVersionNumber(), isBase));
        } else {
            // Otherwise, version must be in cache
            return new CurlyFilePath(
                    localPartitionRootCache + "/" + Curly.getName(version.getVersionNumber(), isBase));
        }
    }
}