gobblin.util.recordcount.IngestionRecordCountProvider.java Source code

Introduction

Here is the source code for gobblin.util.recordcount.IngestionRecordCountProvider.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package gobblin.util.recordcount;

import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;

import com.google.common.base.Preconditions;
import com.google.common.io.Files;

import gobblin.util.RecordCountProvider;

/**
 * Implementation of {@link RecordCountProvider}, which provides record count from file path.
 * The file path should follow the pattern: {Filename}.{RecordCount}.{Extension}.
 * For example, given a file path: "/a/b/c/file.123.avro", the record count will be 123.
 */
public class IngestionRecordCountProvider extends RecordCountProvider {

    private static final String SEPARATOR = ".";

    /**
     * Construct a new file path by appending record count to the filename of the given file path, separated by SEPARATOR.
     * For example, given path: "/a/b/c/file.avro" and record count: 123,
     * the new path returned will be: "/a/b/c/file.123.avro"
     */
    public static String constructFilePath(String oldFilePath, long recordCounts) {
        return new Path(new Path(oldFilePath).getParent(), Files.getNameWithoutExtension(oldFilePath).toString()
                + SEPARATOR + recordCounts + SEPARATOR + Files.getFileExtension(oldFilePath)).toString();
    }

    /**
     * The record count should be the last component before the filename extension.
     */
    @Override
    public long getRecordCount(Path filepath) {
        String[] components = filepath.getName().split(Pattern.quote(SEPARATOR));
        Preconditions.checkArgument(
                components.length >= 2 && StringUtils.isNumeric(components[components.length - 2]),
                String.format("Filename %s does not follow the pattern: FILENAME.RECORDCOUNT.EXTENSION", filepath));
        return Long.parseLong(components[components.length - 2]);
    }
}