Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.aliyun.odps.local.common.utils; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; import com.aliyun.odps.utils.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import com.aliyun.odps.Column; import com.aliyun.odps.OdpsType; import com.aliyun.odps.data.TableInfo; import com.aliyun.odps.local.common.Constants; import com.aliyun.odps.local.common.TableMeta; public class SchemaUtils { private static final Log LOG = LogFactory.getLog(SchemaUtils.class); private static final String SEPERATOR = ","; private static final String DELIMITER = ":"; public static boolean existsSchemaFile(File dir) { return new File(dir, Constants.SCHEMA_FILE).exists(); } public static void generateSchemaFile(TableMeta table, List<Integer> indexes, File dir) { if (table == null || dir == null) { throw new IllegalArgumentException("Missing arguments: table|dir"); } if (StringUtils.isBlank(table.getProjName()) || StringUtils.isBlank(table.getTableName())) { throw new IllegalArgumentException( "Project|table is empty when table.getProjName()|table.getTableName()"); } TableInfo tableInfo = TableInfo.builder().projectName(table.getProjName()).tableName(table.getTableName()) .build(); LOG.info("Start to write table scheme : " + tableInfo + "-->" + dir.getAbsolutePath()); StringBuffer sb = new StringBuffer(); sb.append("project=" + table.getProjName()); sb.append("\n"); sb.append("table=" + table.getTableName()); sb.append("\n"); StringBuffer sb1 = new StringBuffer(); Column[] columns = table.getCols(); int length = indexes == null ? table.getCols().length : indexes.size(); for (int i = 0; i < length; i++) { int index = indexes == null ? i : indexes.get(i); Column col = columns[index]; if (sb1.length() > 0) { sb1.append(","); } sb1.append(col.getName() + ":" + col.getType().toString()); } sb.append("columns=" + sb1.toString()); sb.append("\n"); Column[] partitions = table.getPartitions(); if (partitions != null && partitions.length > 0) { sb1 = new StringBuffer(); for (int i = 0; i < partitions.length; i++) { if (sb1.length() > 0) { sb1.append(","); } sb1.append(partitions[i].getName() + ":" + partitions[i].getType().toString()); } sb.append("partitions=" + sb1.toString()); sb.append("\n"); } dir.mkdirs(); File schemaFile = new File(dir, Constants.SCHEMA_FILE); LOG.info("generate schema file: " + schemaFile.getAbsolutePath()); OutputStream out = null; try { out = new FileOutputStream(schemaFile); String result = sb.toString(); // ??\n result = result.substring(0, result.length() - 1); out.write(result.getBytes("utf-8")); } catch (IOException exception) { throw new RuntimeException(exception); } finally { try { out.close(); } catch (IOException e) { throw new RuntimeException(e); } } LOG.info("Finished to write table scheme : " + tableInfo + "-->" + dir.getAbsolutePath()); } public static TableMeta readSchema(File dir) { if (dir == null || !dir.exists()) { return null; } File schemaFile = new File(dir, Constants.SCHEMA_FILE); BufferedReader br = null; try { br = new BufferedReader(new InputStreamReader(new FileInputStream(schemaFile))); } catch (FileNotFoundException e) { throw new RuntimeException("__schema__ file not exists in direcotry " + dir.getAbsolutePath()); } String line = null; try { line = br.readLine(); } catch (IOException e1) { throw new RuntimeException(e1); } String project = null; String table = null; List<Column> cols = new ArrayList<Column>(); List<Column> parts = new ArrayList<Column>(); while (line != null) { line = line.trim(); if (line.equals("") || line.startsWith("#")) { try { line = br.readLine(); } catch (IOException e) { throw new RuntimeException(e); } continue; } String[] kv = line.split("="); if (kv == null || kv.length != 2 || kv[0] == null || kv[0].trim().isEmpty() || kv[1] == null || kv[1].trim().isEmpty()) { try { line = br.readLine(); } catch (IOException e) { throw new RuntimeException(e); } continue; } kv[0] = kv[0].trim(); kv[1] = kv[1].trim(); if (kv[0].equals("project")) { project = kv[1]; } else if (kv[0].equals("table")) { table = kv[1]; if (table == null || table.trim().isEmpty()) { throw new RuntimeException("Table schema file '_schema_' must include 'table'"); } } else if (kv[0].equals("columns")) { String columns = kv[1]; if (columns == null || columns.trim().isEmpty()) { throw new RuntimeException("Table schema file '_schema_' must include 'columns'"); } String[] ss = columns.split(","); for (int i = 0; i < ss.length; i++) { String[] temp = ss[i].trim().split(":"); if (temp.length == 2) { temp[0] = temp[0].trim(); temp[1] = temp[1].trim(); if (!temp[0].isEmpty() && !temp[1].isEmpty()) { cols.add(new Column(temp[0], OdpsType.valueOf(temp[1].toUpperCase()))); } } } if (cols.size() == 0) { throw new RuntimeException("'columns' in table schema file '_schema_' has invalid value"); } } else if (kv[0].equals("partitions")) { String partitions = kv[1]; if (partitions != null && !partitions.trim().isEmpty()) { String[] ss = partitions.split(","); for (int i = 0; i < ss.length; i++) { String[] temp = ss[i].trim().split(":"); if (temp.length == 2) { temp[0] = temp[0].trim(); temp[1] = temp[1].trim(); if (!temp[0].isEmpty() && !temp[1].isEmpty()) { parts.add(new Column(temp[0], OdpsType.valueOf(temp[1].toUpperCase()))); } } } } } try { line = br.readLine(); } catch (IOException e) { throw new RuntimeException(e); } } try { br.close(); } catch (IOException e) { throw new RuntimeException(e); } return new TableMeta(project, table, cols.toArray(new Column[cols.size()]), parts.toArray(new Column[parts.size()])); } /** * ??? * * @param cols * * @return ?? */ public static String[] getColumnNames(Column[] cols) { String[] names = new String[cols.length]; for (int i = 0; i < cols.length; i++) { names[i] = cols[i].getName(); } return names; } /** * ??? * * @param cols * * @return ?? * @see #fromString(String) */ public static String toString(Column[] cols) { if (cols == null) { return ""; } StringBuilder sb = new StringBuilder(); for (Column c : cols) { if (c == null) { continue; } if (sb.length() > 0) { sb.append(SEPERATOR); } sb.append(c.getName()).append(DELIMITER).append(c.getType().toString()); } return sb.toString(); } }