org.apache.nifi.lookup.SimpleCsvFileLookupService.java Source code

Java tutorial

Introduction

Here is the source code for org.apache.nifi.lookup.SimpleCsvFileLookupService.java

Source

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.nifi.lookup;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;

import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnEnabled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.controller.AbstractControllerService;
import org.apache.nifi.controller.ControllerServiceInitializationContext;
import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.file.monitor.LastModifiedMonitor;
import org.apache.nifi.util.file.monitor.SynchronousFileWatcher;

@Tags({ "lookup", "cache", "enrich", "join", "csv", "reloadable", "key", "value" })
@CapabilityDescription("A reloadable CSV file-based lookup service")
public class SimpleCsvFileLookupService extends AbstractControllerService implements StringLookupService {

    private static final String KEY = "key";

    private static final Set<String> REQUIRED_KEYS = Collections
            .unmodifiableSet(Stream.of(KEY).collect(Collectors.toSet()));

    public static final PropertyDescriptor CSV_FILE = new PropertyDescriptor.Builder().name("csv-file")
            .displayName("CSV File").description("A CSV file.").required(true)
            .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR).expressionLanguageSupported(true).build();

    static final PropertyDescriptor CSV_FORMAT = new PropertyDescriptor.Builder().name("CSV Format").description(
            "Specifies which \"format\" the CSV data is in, or specifies if custom formatting should be used.")
            .expressionLanguageSupported(false)
            .allowableValues(Arrays.asList(CSVFormat.Predefined.values()).stream().map(e -> e.toString())
                    .collect(Collectors.toSet()))
            .defaultValue(CSVFormat.Predefined.Default.toString()).required(true).build();

    public static final PropertyDescriptor LOOKUP_KEY_COLUMN = new PropertyDescriptor.Builder()
            .name("lookup-key-column").displayName("Lookup Key Column").description("Lookup key column.")
            .required(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true)
            .build();

    public static final PropertyDescriptor LOOKUP_VALUE_COLUMN = new PropertyDescriptor.Builder()
            .name("lookup-value-column").displayName("Lookup Value Column").description("Lookup value column.")
            .required(true).addValidator(StandardValidators.NON_EMPTY_VALIDATOR).expressionLanguageSupported(true)
            .build();

    public static final PropertyDescriptor IGNORE_DUPLICATES = new PropertyDescriptor.Builder()
            .name("ignore-duplicates").displayName("Ignore Duplicates")
            .description("Ignore duplicate keys for records in the CSV file.")
            .addValidator(StandardValidators.BOOLEAN_VALIDATOR).allowableValues("true", "false")
            .defaultValue("true").required(true).build();

    private List<PropertyDescriptor> properties;

    private volatile ConcurrentMap<String, String> cache;

    private volatile String csvFile;

    private volatile CSVFormat csvFormat;

    private volatile String lookupKeyColumn;

    private volatile String lookupValueColumn;

    private volatile boolean ignoreDuplicates;

    private volatile SynchronousFileWatcher watcher;

    private final ReentrantLock lock = new ReentrantLock();

    private void loadCache() throws IllegalStateException, IOException {
        if (lock.tryLock()) {
            try {
                final ComponentLog logger = getLogger();
                if (logger.isDebugEnabled()) {
                    logger.debug("Loading lookup table from file: " + csvFile);
                }

                final Map<String, String> properties = new HashMap<>();
                final FileReader reader = new FileReader(csvFile);
                final Iterable<CSVRecord> records = csvFormat.withFirstRecordAsHeader().parse(reader);
                for (final CSVRecord record : records) {
                    final String key = record.get(lookupKeyColumn);
                    final String value = record.get(lookupValueColumn);
                    if (StringUtils.isBlank(key)) {
                        throw new IllegalStateException("Empty lookup key encountered in: " + csvFile);
                    } else if (!ignoreDuplicates && properties.containsKey(key)) {
                        throw new IllegalStateException(
                                "Duplicate lookup key encountered: " + key + " in " + csvFile);
                    } else if (ignoreDuplicates && properties.containsKey(key)) {
                        logger.warn("Duplicate lookup key encountered: {} in {}", new Object[] { key, csvFile });
                    }
                    properties.put(key, value);
                }

                this.cache = new ConcurrentHashMap<>(properties);

                if (cache.isEmpty()) {
                    logger.warn("Lookup table is empty after reading file: " + csvFile);
                }
            } finally {
                lock.unlock();
            }
        }
    }

    @Override
    protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
        return properties;
    }

    @Override
    protected void init(final ControllerServiceInitializationContext context) throws InitializationException {
        final List<PropertyDescriptor> properties = new ArrayList<>();
        properties.add(CSV_FILE);
        properties.add(CSV_FORMAT);
        properties.add(LOOKUP_KEY_COLUMN);
        properties.add(LOOKUP_VALUE_COLUMN);
        properties.add(IGNORE_DUPLICATES);
        this.properties = Collections.unmodifiableList(properties);
    }

    @OnEnabled
    public void onEnabled(final ConfigurationContext context)
            throws InitializationException, IOException, FileNotFoundException {
        this.csvFile = context.getProperty(CSV_FILE).getValue();
        this.csvFormat = CSVFormat.Predefined.valueOf(context.getProperty(CSV_FORMAT).getValue()).getFormat();
        this.lookupKeyColumn = context.getProperty(LOOKUP_KEY_COLUMN).getValue();
        this.lookupValueColumn = context.getProperty(LOOKUP_VALUE_COLUMN).getValue();
        this.ignoreDuplicates = context.getProperty(IGNORE_DUPLICATES).asBoolean();
        this.watcher = new SynchronousFileWatcher(Paths.get(csvFile), new LastModifiedMonitor(), 30000L);
        try {
            loadCache();
        } catch (final IllegalStateException e) {
            throw new InitializationException(e.getMessage(), e);
        }
    }

    @Override
    public Optional<String> lookup(final Map<String, Object> coordinates) throws LookupFailureException {
        if (coordinates == null) {
            return Optional.empty();
        }

        final String key = coordinates.get(KEY).toString();
        if (StringUtils.isBlank(key)) {
            return Optional.empty();
        }

        try {
            if (watcher != null && watcher.checkAndReset()) {
                loadCache();
            }
        } catch (final IllegalStateException | IOException e) {
            throw new LookupFailureException(e.getMessage(), e);
        }

        return Optional.ofNullable(cache.get(key));
    }

    @Override
    public Set<String> getRequiredKeys() {
        return REQUIRED_KEYS;
    }

}