Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tajo.storage.http; import com.google.common.collect.Lists; import net.minidev.json.JSONObject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.storage.FormatProperty; import org.apache.tajo.storage.StorageProperty; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.Fragment; import org.jboss.netty.handler.codec.http.HttpHeaders.Names; import javax.annotation.Nullable; import java.io.FileNotFoundException; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.util.List; import java.util.Optional; /** * Example read-only tablespace for HTTP protocol. * * An example table can be created by using the following SQL query. * * CREATE TABLE http_test (*) TABLESPACE http_example USING ex_http_json WITH ('path'='2015-01-01-15.json.gz', * 'compression.codec'='org.apache.hadoop.io.compress.GzipCodec); */ public class ExampleHttpFileTablespace extends Tablespace { private static final Log LOG = LogFactory.getLog(ExampleHttpFileTablespace.class); static final String PATH = "path"; ////////////////////////////////////////////////////////////////////////////////////////////////// // Tablespace properties ////////////////////////////////////////////////////////////////////////////////////////////////// private static final StorageProperty STORAGE_PROPERTY = new StorageProperty(BuiltinStorages.JSON, // default format is json false, // is not movable false, // is not writable true, // allow arbitrary path false // doesn't provide metadata ); private static final FormatProperty FORMAT_PROPERTY = new FormatProperty(false, // doesn't support insert false, // doesn't support direct insert false // doesn't support result staging ); public ExampleHttpFileTablespace(String name, URI uri, JSONObject config) { super(name, uri, config); LOG.info("ExampleHttpFileTablespace is initialized for " + uri); } @Override protected void storageInit() throws IOException { // Add initialization code for your tablespace } @Override public long getTableVolume(TableDesc table, Optional<EvalNode> notUsed) { HttpURLConnection connection = null; try { connection = (HttpURLConnection) new URL(table.getUri().toASCIIString()).openConnection(); connection.setRequestMethod("HEAD"); connection.connect(); return connection.getHeaderFieldLong(Names.CONTENT_LENGTH, -1); } catch (IOException e) { throw new TajoInternalError(e); } finally { if (connection != null) { connection.disconnect(); } } } @Override public URI getRootUri() { return uri; } @Override public URI getTableUri(TableMeta meta, String databaseName, String tableName) { String tablespaceUriString = uri.toASCIIString(); String tablePath = meta.getProperty(PATH); if (!tablespaceUriString.endsWith("/") && !tablePath.startsWith("/")) { tablePath = "/" + tablePath; } return URI.create(tablespaceUriString + tablePath); } @Override public List<Fragment> getSplits(String inputSourceId, TableDesc tableDesc, boolean requireSort, @Nullable EvalNode filterCondition) throws IOException, TajoException { // getSplits() should return multiple fragments for distributed processing of a large data. // This example tablespace returns only one fragment for the whole data for simplicity, // but this may significantly increase the query processing time. long tableVolume = getTableVolume(tableDesc, Optional.empty()); return Lists.newArrayList(new ExampleHttpFileFragment(tableDesc.getUri(), inputSourceId, 0, tableVolume)); } @Override public StorageProperty getProperty() { return STORAGE_PROPERTY; } @Override public FormatProperty getFormatProperty(TableMeta meta) { return FORMAT_PROPERTY; } @Override public void close() { // do nothing } @Override public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) throws IOException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public void verifySchemaToWrite(TableDesc tableDesc, Schema outSchema) throws TajoException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public void createTable(TableDesc table, boolean ifNotExists) throws TajoException, IOException { HttpURLConnection connection = null; try { connection = (HttpURLConnection) new URL(table.getUri().toASCIIString()).openConnection(); connection.setRequestMethod("HEAD"); connection.connect(); if (connection.getResponseCode() == 404) { throw new FileNotFoundException(); } } catch (IOException e) { throw new TajoInternalError(e); } finally { if (connection != null) { connection.disconnect(); } } } @Override public void purgeTable(TableDesc tableDesc) throws IOException, TajoException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public void prepareTable(LogicalNode node) throws IOException, TajoException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public Path commitTable(OverridableConf queryContext, ExecutionBlockId finalEbId, LogicalPlan plan, Schema schema, TableDesc tableDesc) throws IOException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public void rollbackTable(LogicalNode node) throws IOException, TajoException { throw new TajoRuntimeException(new UnsupportedException()); } @Override public URI getStagingUri(OverridableConf context, String queryId, TableMeta meta) throws IOException { throw new TajoRuntimeException(new UnsupportedException()); } }