com.yahoo.sql4d.indexeragent.sql.SqlFileSniffer.java Source code

Introduction

Here is the source code for com.yahoo.sql4d.indexeragent.sql.SqlFileSniffer.java

Source

/**
 * Copyright 2014 Yahoo! Inc. Licensed under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with the
 * License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
 * or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the specific language
 * governing permissions and limitations under the License. See accompanying
 * LICENSE file.
 */
package com.yahoo.sql4d.indexeragent.sql;

import com.yahoo.sql4d.DCompiler;
import com.yahoo.sql4d.InsertProgram;
import com.yahoo.sql4d.Program;
import static com.yahoo.sql4d.indexeragent.Agent.db;
import com.yahoo.sql4d.indexeragent.meta.JobFreq;
import com.yahoo.sql4d.indexeragent.meta.JobStatus;
import com.yahoo.sql4d.indexeragent.meta.beans.DataSource;
import com.yahoo.sql4d.indexeragent.util.*;
import com.yahoo.sql4d.insert.BasicInsertMeta;
import com.yahoo.sql4d.insert.BatchInsertMeta;
import com.yahoo.sql4d.query.nodes.Interval;
import static com.yahoo.sql4d.utils.TimeUtils.getDateTime;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Observes change to sql files and calls the corresponding event handlers. The 
 * callbacks can also be called directly in business logic as it is stateless and thread safe.
 * @author srikalyan
 */
public class SqlFileSniffer extends FileSniffer {
    private static final Logger log = LoggerFactory.getLogger(SqlFileSniffer.class);

    public SqlFileSniffer(String pathToSniff) {
        super(pathToSniff, ".sql");
    }

    @Override
    public void onCreate(Path path) {
        addOrUpdateTable(path.toFile());
    }

    @Override
    public void onDelete(Path path) {
        String fileNameOnly = path.getFileName().toString().replaceAll(".sql", "");
        DataSource existingDs = db().getDataSource(fileNameOnly);
        if (existingDs != null) {
            db().removeDataSource(existingDs);
        }
    }

    @Override
    public void onModify(Path path) {
        addOrUpdateTable(path.toFile());
    }

    /**
     * If a sql file is added/updated this can be called.
     * @param file 
     */
    private void addOrUpdateTable(File file) {
        try {
            String insertStmnt = FileUtils.readFileToString(file);
            DataSource ds = processSql(insertStmnt);
            DataSource existingDs = db().getDataSource(ds.getName());
            if (existingDs == null) {//We never saw this table before.
                db().addDataSource(ds);
                return;
            }
            // If startTime/endTime/templatePath changed then we should update the table.
            if (existingDs.getStartTime() != ds.getStartTime() || existingDs.getEndTime() != ds.getEndTime()
                    || !existingDs.getTemplatePath().equals(ds.getTemplatePath())) {
                // We have to reuse the existing entity bean(else update will fail).
                existingDs.updateFrom(ds);
                db().updateDataSource(existingDs);
            }
        } catch (IOException ex) {
            log.error("Exception while loading druid sql file {} : {}", file, ex);
        }
    }

    /**
     * Creates DataSource for the given Sql insert statement.
     * @param insertStmntStr
     * @return 
     */
    private DataSource processSql(String insertStmntStr) {
        Program pgm = DCompiler.compileSql(insertStmntStr);
        if (!(pgm instanceof InsertProgram)) {
            log.error("Ignoring program {} . Only inserts are supported", insertStmntStr);
            return null;
        }
        InsertProgram insertPgm = (InsertProgram) pgm;
        DataSource ds = new DataSource();
        switch (insertPgm.getStmntType()) {
        case INSERT:
            BasicInsertMeta stmt = (BasicInsertMeta) insertPgm.nthStmnt(0);
            Interval interval = stmt.granularitySpec.interval;
            // Round to nearest hour(zero out the mins, secs and millis)
            long startTime = getDateTime(interval.startTime).withMinuteOfHour(0).withSecondOfMinute(0)
                    .withMillisOfSecond(0).getMillis();
            ds.setName(stmt.dataSource).setDelimiter(stmt.delimiter).setListDelimiter(stmt.listDelimiter)
                    .setTemplatePath(stmt.dataPath).setStartTime(startTime).setSpinFromTime(startTime)
                    .setFrequency(JobFreq.valueOf(stmt.granularitySpec.gran))
                    .setEndTime(getDateTime(interval.endTime).getMillis()).setStatus(JobStatus.not_done)
                    .setTemplateSql(templatizeSql(insertStmntStr, interval));
            break;
        case INSERT_HADOOP:
            BatchInsertMeta bStmt = (BatchInsertMeta) insertPgm.nthStmnt(0);
            Interval bInterval = bStmt.granularitySpec.interval;
            // Round to nearest hour(zero out the mins, secs and millis)
            long startBTime = getDateTime(bInterval.startTime).withMinuteOfHour(0).withSecondOfMinute(0)
                    .withMillisOfSecond(0).getMillis();
            ds.setName(bStmt.dataSource).setDelimiter(bStmt.delimiter).setListDelimiter(bStmt.listDelimiter)
                    .setTemplatePath(bStmt.inputSpec.getRawPath()).setStartTime(startBTime)
                    .setSpinFromTime(startBTime).setFrequency(JobFreq.valueOf(bStmt.granularitySpec.gran))
                    .setEndTime(getDateTime(bInterval.endTime).getMillis()).setStatus(JobStatus.not_done)
                    .setTemplateSql(templatizeSql(insertStmntStr, bInterval));
            break;
        case INSERT_REALTIME:
            log.error("Realtime insert currently unsupported {}", pgm);
            return null;
        }
        return ds;
    }

    private String templatizeSql(String payload, Interval interval) {
        return payload.replaceAll(interval.startTime, ":startTime").replaceAll(interval.endTime, ":endTime");
    }
}