Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.sqoop.manager.oracle; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.sql.Connection; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.log4j.Category; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.sqoop.manager.oracle.OraOopOutputFormatInsert.InsertMode; import org.apache.sqoop.manager.oracle.OraOopOutputFormatUpdate.UpdateMode; import com.cloudera.sqoop.Sqoop; import com.cloudera.sqoop.SqoopOptions; import com.cloudera.sqoop.mapreduce.ExportJobBase; /** * Utilities used by OraOop. */ public final class OraOopUtilities { private OraOopUtilities() { } private static String currentSessionUser = null; /** * Used for testing purposes - can get OraOop to call a class to run a report * on various performance metrics. */ public static class OraOopStatsReports { private String csvReport; private String performanceReport; public String getPerformanceReport() { return performanceReport; } public void setPerformanceReport(String newPerformanceReport) { this.performanceReport = newPerformanceReport; } public String getCsvReport() { return csvReport; } public void setCsvReport(String newCsvReport) { this.csvReport = newCsvReport; } } protected static final OraOopLog LOG = OraOopLogFactory.getLog(OraOopUtilities.class.getName()); public static List<String> copyStringList(List<String> list) { List<String> result = new ArrayList<String>(list.size()); result.addAll(list); return result; } public static OracleTable decodeOracleTableName(String oracleConnectionUserName, String tableStr) { String tableOwner; String tableName; // These are the possibilities for double-quote location... // table // "table" // schema.table // schema."table" // "schema".table // "schema"."table" String[] tableStrings = tableStr.split("\""); if (oracleConnectionUserName == null) { oracleConnectionUserName = currentSessionUser; } switch (tableStrings.length) { case 1: // <- table or schema.table tableStrings = tableStr.split("\\."); switch (tableStrings.length) { case 1: // <- No period tableOwner = oracleConnectionUserName.toUpperCase(); tableName = tableStrings[0].toUpperCase(); break; case 2: // <- 1 period tableOwner = tableStrings[0].toUpperCase(); tableName = tableStrings[1].toUpperCase(); break; default: LOG.debug(String.format("Unable to decode the table name (displayed in " + "double quotes): \"%s\"", tableStr)); throw new RuntimeException(String.format("Unable to decode the table name: %s", tableStr)); } break; case 2: // <- "table" or schema."table" if (tableStrings[0] == null || tableStrings[0].isEmpty()) { tableOwner = oracleConnectionUserName.toUpperCase(); } else { tableOwner = tableStrings[0].toUpperCase(); // Remove the "." from the end of the schema name... if (tableOwner.endsWith(".")) { tableOwner = tableOwner.substring(0, tableOwner.length() - 1); } } tableName = tableStrings[1]; break; case 3: // <- "schema".table tableOwner = tableStrings[1]; tableName = tableStrings[2].toUpperCase(); // Remove the "." from the start of the table name... if (tableName.startsWith(".")) { tableName = tableName.substring(1, tableName.length()); } break; case 4: // <- "schema"."table" tableOwner = tableStrings[1]; tableName = tableStrings[3]; break; default: LOG.debug(String.format("Unable to decode the table name (displayed in double " + "quotes): \"%s\"", tableStr)); throw new RuntimeException(String.format("Unable to decode the table name: %s", tableStr)); } OracleTable result = new OracleTable(tableOwner, tableName); return result; } public static OracleTable decodeOracleTableName(String oracleConnectionUserName, String tableStr, org.apache.hadoop.conf.Configuration conf) { OracleTable result = new OracleTable(); // Have we already determined the answer to this question?... if (conf != null) { String tableOwner = conf.get(OraOopConstants.ORAOOP_TABLE_OWNER); String tableName = conf.get(OraOopConstants.ORAOOP_TABLE_NAME); result = new OracleTable(tableOwner, tableName); } // If we couldn't look up the answer, then determine it now... if (result.getSchema() == null || result.getName() == null) { result = decodeOracleTableName(oracleConnectionUserName, tableStr); LOG.debug(String.format( "The Oracle table context has been derived from:\n" + "\toracleConnectionUserName = %s\n" + "\ttableStr = %s\n" + "\tas:\n" + "\towner : %s\n" + "\ttable : %s", oracleConnectionUserName, tableStr, result.getSchema(), result.getName())); // Save the answer for next time... if (conf != null) { conf.set(OraOopConstants.ORAOOP_TABLE_OWNER, result.getSchema()); conf.set(OraOopConstants.ORAOOP_TABLE_NAME, result.getName()); } } return result; } public static boolean oracleJdbcUrlGenerationDisabled(org.apache.hadoop.conf.Configuration conf) { return conf.getBoolean(OraOopConstants.ORAOOP_JDBC_URL_VERBATIM, false); } public static boolean userWantsOracleSessionStatisticsReports(org.apache.hadoop.conf.Configuration conf) { return conf.getBoolean(OraOopConstants.ORAOOP_REPORT_SESSION_STATISTICS, false); } public static boolean enableDebugLoggingIfRequired(org.apache.hadoop.conf.Configuration conf) { boolean result = false; try { Level desiredOraOopLoggingLevel = Level.toLevel(conf.get(OraOopConstants.ORAOOP_LOGGING_LEVEL), Level.INFO); Level sqoopLogLevel = Logger.getLogger(Sqoop.class.getName()).getParent().getLevel(); if (desiredOraOopLoggingLevel == Level.DEBUG || desiredOraOopLoggingLevel == Level.ALL || sqoopLogLevel == Level.DEBUG || sqoopLogLevel == Level.ALL) { Category oraOopLogger = Logger.getLogger(OraOopManagerFactory.class.getName()).getParent(); oraOopLogger.setLevel(Level.DEBUG); LOG.debug("Enabled OraOop debug logging."); result = true; conf.set(OraOopConstants.ORAOOP_LOGGING_LEVEL, Level.DEBUG.toString()); } } catch (Exception ex) { LOG.error(String.format("Unable to determine whether debug logging should be enabled.\n%s", getFullExceptionMessage(ex))); } return result; } public static String generateDataChunkId(int fileId, int fileBatch) { StringBuilder sb = new StringBuilder(); return sb.append(fileId).append("_").append(fileBatch).toString(); } public static String getCurrentMethodName() { StackTraceElement[] stackTraceElements = (new Throwable()).getStackTrace(); return String.format("%s()", stackTraceElements[1].getMethodName()); } public static String[] getDuplicatedStringArrayValues(String[] list, boolean ignoreCase) { if (list == null) { throw new IllegalArgumentException("The list argument cannot be null"); } ArrayList<String> duplicates = new ArrayList<String>(); for (int idx1 = 0; idx1 < list.length - 1; idx1++) { for (int idx2 = idx1 + 1; idx2 < list.length; idx2++) { if (list[idx1].equals(list[idx2])) { // If c is a duplicate of both a & b, don't add c to the list twice... if (!duplicates.contains(list[idx2])) { duplicates.add(list[idx2]); } } else if (ignoreCase && list[idx1].equalsIgnoreCase((list[idx2]))) { // If c is a duplicate of both a & b, don't add c to the list twice... if (stringListIndexOf(duplicates, list[idx2], ignoreCase) == -1) { duplicates.add(list[idx2]); } } } } return duplicates.toArray(new String[duplicates.size()]); } public static String getFullExceptionMessage(Exception ex) { ByteArrayOutputStream arrayStream = new ByteArrayOutputStream(); PrintStream printStream = new PrintStream(arrayStream); ex.printStackTrace(printStream); return arrayStream.toString(); } public static int getMinNumberOfImportMappersAcceptedByOraOop(org.apache.hadoop.conf.Configuration conf) { return conf.getInt(OraOopConstants.ORAOOP_MIN_IMPORT_MAPPERS, OraOopConstants.MIN_NUM_IMPORT_MAPPERS_ACCEPTED_BY_ORAOOP); } public static int getMinAppendValuesBatchSize(org.apache.hadoop.conf.Configuration conf) { return conf.getInt(OraOopConstants.ORAOOP_MIN_APPEND_VALUES_BATCH_SIZE, OraOopConstants.ORAOOP_MIN_APPEND_VALUES_BATCH_SIZE_DEFAULT); } public static int getMinNumberOfExportMappersAcceptedByOraOop(org.apache.hadoop.conf.Configuration conf) { return conf.getInt(OraOopConstants.ORAOOP_MIN_EXPORT_MAPPERS, OraOopConstants.MIN_NUM_EXPORT_MAPPERS_ACCEPTED_BY_ORAOOP); } public static int getMinNumberOfOracleRacActiveInstancesForDynamicJdbcUrlUse( org.apache.hadoop.conf.Configuration conf) { return conf.getInt(OraOopConstants.ORAOOP_MIN_RAC_ACTIVE_INSTANCES, OraOopConstants.MIN_NUM_RAC_ACTIVE_INSTANCES_FOR_DYNAMIC_JDBC_URLS); } public static int getNumberOfDataChunksPerOracleDataFile(int desiredNumberOfMappers, org.apache.hadoop.conf.Configuration conf) { final String MAPPER_MULTIPLIER = "oraoop.datachunk.mapper.multiplier"; final String RESULT_INCREMENT = "oraoop.datachunk.result.increment"; int numberToMultiplyMappersBy = conf.getInt(MAPPER_MULTIPLIER, 2); int numberToIncrementResultBy = conf.getInt(RESULT_INCREMENT, 1); // The number of chunks generated will *not* be a multiple of the number of // splits, // to ensure that each split doesn't always get data from the start of each // data-file... int numberOfDataChunksPerOracleDataFile = (desiredNumberOfMappers * numberToMultiplyMappersBy) + numberToIncrementResultBy; LOG.debug( String.format("%s:\n" + "\t%s=%d\n" + "\t%s=%d\n" + "\tdesiredNumberOfMappers=%d\n" + "\tresult=%d", getCurrentMethodName(), MAPPER_MULTIPLIER, numberToMultiplyMappersBy, RESULT_INCREMENT, numberToIncrementResultBy, desiredNumberOfMappers, numberOfDataChunksPerOracleDataFile)); return numberOfDataChunksPerOracleDataFile; } public static OraOopConstants.OraOopOracleDataChunkMethod getOraOopOracleDataChunkMethod(Configuration conf) { if (conf == null) { throw new IllegalArgumentException("The conf argument cannot be null"); } String strMethod = conf.get(OraOopConstants.ORAOOP_ORACLE_DATA_CHUNK_METHOD); if (strMethod == null) { return OraOopConstants.ORAOOP_ORACLE_DATA_CHUNK_METHOD_DEFAULT; } OraOopConstants.OraOopOracleDataChunkMethod result; try { strMethod = strMethod.toUpperCase().trim(); result = OraOopConstants.OraOopOracleDataChunkMethod.valueOf(strMethod); } catch (IllegalArgumentException ex) { result = OraOopConstants.ORAOOP_ORACLE_DATA_CHUNK_METHOD_DEFAULT; LOG.error("An invalid value of \"" + strMethod + "\" was specified for the \"" + OraOopConstants.ORAOOP_ORACLE_DATA_CHUNK_METHOD + "\" configuration property value.\n" + "\tThe default value of " + OraOopConstants.ORAOOP_ORACLE_DATA_CHUNK_METHOD_DEFAULT + " will be used."); } return result; } public static OraOopConstants.OraOopOracleBlockToSplitAllocationMethod getOraOopOracleBlockToSplitAllocationMethod( org.apache.hadoop.conf.Configuration conf, OraOopConstants.OraOopOracleBlockToSplitAllocationMethod defaultMethod) { if (conf == null) { throw new IllegalArgumentException("The conf argument cannot be null"); } String strMethod = conf.get(OraOopConstants.ORAOOP_ORACLE_BLOCK_TO_SPLIT_ALLOCATION_METHOD); if (strMethod == null) { return defaultMethod; } OraOopConstants.OraOopOracleBlockToSplitAllocationMethod result; try { strMethod = strMethod.toUpperCase().trim(); result = OraOopConstants.OraOopOracleBlockToSplitAllocationMethod.valueOf(strMethod); } catch (IllegalArgumentException ex) { result = defaultMethod; String errorMsg = String.format( "An invalid value of \"%s\" was specified for the \"%s\" " + "configuration property value.\n" + "\tValid values are: %s\n" + "\tThe default value of %s will be used.", strMethod, OraOopConstants.ORAOOP_ORACLE_BLOCK_TO_SPLIT_ALLOCATION_METHOD, getOraOopOracleBlockToSplitAllocationMethods(), defaultMethod.name()); LOG.error(errorMsg); } return result; } private static String getOraOopOracleBlockToSplitAllocationMethods() { OraOopConstants.OraOopOracleBlockToSplitAllocationMethod[] values = OraOopConstants.OraOopOracleBlockToSplitAllocationMethod .values(); StringBuilder result = new StringBuilder((2 * values.length) - 1); // <- Include capacity // for commas for (int idx = 0; idx < values.length; idx++) { OraOopConstants.OraOopOracleBlockToSplitAllocationMethod value = values[idx]; if (idx > 0) { result.append(" or "); } result.append(value.name()); } return result.toString(); } public static OraOopConstants.OraOopTableImportWhereClauseLocation getOraOopTableImportWhereClauseLocation( org.apache.hadoop.conf.Configuration conf, OraOopConstants.OraOopTableImportWhereClauseLocation defaultLocation) { if (conf == null) { throw new IllegalArgumentException("The conf argument cannot be null"); } String strLocation = conf.get(OraOopConstants.ORAOOP_TABLE_IMPORT_WHERE_CLAUSE_LOCATION); if (strLocation == null) { return defaultLocation; } OraOopConstants.OraOopTableImportWhereClauseLocation result; try { strLocation = strLocation.toUpperCase().trim(); result = OraOopConstants.OraOopTableImportWhereClauseLocation.valueOf(strLocation); } catch (IllegalArgumentException ex) { result = defaultLocation; String errorMsg = String.format( "An invalid value of \"%s\"was specified for the \"%s\" " + "configuration property value.\n" + "\tValid values are: %s\n" + "\tThe default value of %s will be used.", strLocation, OraOopConstants.ORAOOP_TABLE_IMPORT_WHERE_CLAUSE_LOCATION, getOraOopTableImportWhereClauseLocations(), defaultLocation.name()); LOG.error(errorMsg); } return result; } private static String getOraOopTableImportWhereClauseLocations() { OraOopConstants.OraOopTableImportWhereClauseLocation[] locationValues = OraOopConstants.OraOopTableImportWhereClauseLocation .values(); StringBuilder result = new StringBuilder((2 * locationValues.length) - 1); // <- Include // capacity for // commas for (int idx = 0; idx < locationValues.length; idx++) { OraOopConstants.OraOopTableImportWhereClauseLocation locationValue = locationValues[idx]; if (idx > 0) { result.append(" or "); } result.append(locationValue.name()); } return result.toString(); } public static String getOutputDirectory(org.apache.hadoop.conf.Configuration conf) { String workingDir = conf.get("mapred.working.dir"); String outputDir = conf.get("mapred.output.dir"); return workingDir + "/" + outputDir; } public static String padLeft(String s, int n) { return StringUtils.leftPad(s, n); } public static String padRight(String s, int n) { return StringUtils.rightPad(s, n); } public static String replaceConfigurationExpression(String str, org.apache.hadoop.conf.Configuration conf) { int startPos = str.indexOf('{'); int endPos = str.indexOf('}'); // Example: // alter session set timezone = '{oracle.sessionTimeZone|GMT}'; if (startPos == -1 || endPos == -1) { return str; } String configName = null; String defaultValue = null; String expression = str.substring(startPos + 1, endPos); int defaultValuePos = expression.indexOf('|'); if (defaultValuePos == -1) { // return expression; configName = expression; } else { configName = expression.substring(0, defaultValuePos); defaultValue = expression.substring(defaultValuePos + 1); } if (defaultValue == null) { defaultValue = ""; } String configValue = conf.get(configName); if (configValue == null) { configValue = defaultValue; } String result = str.replace(String.format("{%s}", expression), configValue); LOG.debug(String.format("The expression:\n%s\nwas replaced with:\n%s", str, result)); // Recurse to evaluate any other expressions... result = replaceConfigurationExpression(result, conf); return result; } public static boolean stackContainsClass(String className) { StackTraceElement[] stackTraceElements = (new Throwable()).getStackTrace(); for (StackTraceElement stackTraceElement : stackTraceElements) { if (stackTraceElement.getClassName().equalsIgnoreCase(className)) { return true; } } return false; } public static Object startSessionSnapshot(Connection connection) { Object result = null; try { Class<?> oraOopOraStatsClass = Class.forName("quest.com.oraOop.oracleStats.OraOopOraStats"); Method startSnapshotMethod = oraOopOraStatsClass.getMethod("startSnapshot", Connection.class); if (connection != null) { result = startSnapshotMethod.invoke(null, connection); } } catch (ClassNotFoundException ex) { throw new RuntimeException(ex); } catch (NoSuchMethodException ex) { throw new RuntimeException(ex); } catch (InvocationTargetException ex) { throw new RuntimeException(ex); } catch (IllegalAccessException ex) { throw new RuntimeException(ex); } return result; } public static OraOopStatsReports stopSessionSnapshot(Object oraOopOraStats) { OraOopStatsReports result = new OraOopStatsReports(); if (oraOopOraStats == null) { return result; } try { Class<?> oraOopOraStatsClass = Class.forName("quest.com.oraOop.oracleStats.OraOopOraStats"); Method finalizeSnapshotMethod = oraOopOraStatsClass.getMethod("finalizeSnapshot", (Class<?>[]) null); finalizeSnapshotMethod.invoke(oraOopOraStats, (Object[]) null); Method performanceReportCsvMethod = oraOopOraStatsClass.getMethod("getStatisticsCSV", (Class<?>[]) null); result.setCsvReport((String) performanceReportCsvMethod.invoke(oraOopOraStats, (Object[]) null)); Method performanceReportMethod = oraOopOraStatsClass.getMethod("performanceReport", (Class<?>[]) null); result.setPerformanceReport((String) performanceReportMethod.invoke(oraOopOraStats, (Object[]) null)); } catch (ClassNotFoundException ex) { throw new RuntimeException(ex); } catch (NoSuchMethodException ex) { throw new RuntimeException(ex); } catch (InvocationTargetException ex) { throw new RuntimeException(ex); } catch (IllegalAccessException ex) { throw new RuntimeException(ex); } return result; } public static boolean stringArrayContains(String[] list, String value, boolean ignoreCase) { return stringArrayIndexOf(list, value, ignoreCase) > -1; } public static int stringArrayIndexOf(String[] list, String value, boolean ignoreCase) { for (int idx = 0; idx < list.length; idx++) { if (list[idx].equals(value)) { return idx; } if (ignoreCase && list[idx].equalsIgnoreCase(value)) { return idx; } } return -1; } public static String stringArrayToCSV(String[] list) { return stringArrayToCSV(list, ""); } public static String stringArrayToCSV(String[] list, String encloseValuesWith) { StringBuilder result = new StringBuilder((list.length * 2) - 1); for (int idx = 0; idx < list.length; idx++) { if (idx > 0) { result.append(","); } result.append(String.format("%1$s%2$s%1$s", encloseValuesWith, list[idx])); } return result.toString(); } public static int stringListIndexOf(List<String> list, String value, boolean ignoreCase) { for (int idx = 0; idx < list.size(); idx++) { if (list.get(idx).equals(value)) { return idx; } if (ignoreCase && list.get(idx).equalsIgnoreCase(value)) { return idx; } } return -1; } public static void writeOutputFile(org.apache.hadoop.conf.Configuration conf, String fileName, String fileText) { Path uniqueFileName = null; try { FileSystem fileSystem = FileSystem.get(conf); // NOTE: This code is not thread-safe. // i.e. A race-condition could still cause this code to 'fail'. int suffix = 0; String fileNameTemplate = fileName + "%s"; while (true) { uniqueFileName = new Path(getOutputDirectory(conf), String.format(fileNameTemplate, suffix == 0 ? "" : String.format(" (%d)", suffix))); if (!fileSystem.exists(uniqueFileName)) { break; } suffix++; } FSDataOutputStream outputStream = fileSystem.create(uniqueFileName, false); if (fileText != null) { outputStream.writeBytes(fileText); } outputStream.flush(); outputStream.close(); } catch (IOException ex) { LOG.error(String.format("Error attempting to write the file %s\n" + "%s", (uniqueFileName == null ? "null" : uniqueFileName.toUri()), getFullExceptionMessage(ex))); } } /** * Class to wrap details about Oracle connection string. */ public static class JdbcOracleThinConnection { private String host; private int port; private String sid; private String service; private String tnsName; public JdbcOracleThinConnection(String host, int port, String sid, String service, String tnsName) { this.host = host; this.port = port; this.sid = sid; this.service = service; this.tnsName = tnsName; } @Override public String toString() { // Use tnsName if it is available if (this.tnsName != null && !this.tnsName.isEmpty()) { return String.format("jdbc:oracle:thin:@%s", tnsName); } // Use the SID if it's available... if (this.sid != null && !this.sid.isEmpty()) { return String.format("jdbc:oracle:thin:@%s:%d:%s", this.host, this.port, this.sid); } // Otherwise, use the SERVICE. Note that the service is prefixed by "/", // not by ":"... if (this.service != null && !this.service.isEmpty()) { return String.format("jdbc:oracle:thin:@%s:%d/%s", this.host, this.port, this.service); } throw new RuntimeException( "Unable to generate a JDBC URL, as no TNS name, SID or SERVICE " + "has been provided."); } public String getHost() { return host; } public int getPort() { return port; } public String getSid() { return sid; } public String getService() { return service; } public String getTnsName() { return tnsName; } } /** * Thrown if the Oracle connection string cannot be parsed. */ public static class JdbcOracleThinConnectionParsingError extends Exception { private static final long serialVersionUID = 1559860600099354233L; public JdbcOracleThinConnectionParsingError(String message) { super(message); } public JdbcOracleThinConnectionParsingError(String message, Throwable cause) { super(message, cause); } public JdbcOracleThinConnectionParsingError(Throwable cause) { super(cause); } } public static String getOracleServiceName(org.apache.hadoop.conf.Configuration conf) { return conf.get(OraOopConstants.ORAOOP_ORACLE_RAC_SERVICE_NAME, ""); } public static String generateOracleSidJdbcUrl(String hostName, int port, String sid) { return String.format( "jdbc:oracle:thin:@(DESCRIPTION=" + "(ADDRESS_LIST=" + "(ADDRESS=(PROTOCOL=TCP)(HOST=%s)(PORT=%d))" + ")" + "(CONNECT_DATA=(SERVER=DEDICATED)(SID=%s))" + ")", hostName, port, sid); } public static String generateOracleServiceNameJdbcUrl(String hostName, int port, String serviceName) { return String.format( "jdbc:oracle:thin:@(DESCRIPTION=" + "(ADDRESS_LIST=" + "(ADDRESS=(PROTOCOL=TCP)(HOST=%s)(PORT=%d))" + ")" + "(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=%s))" + ")", hostName, port, serviceName); } public static String generateOracleTnsNameJdbcUrl(String tnsName) { return String.format("jdbc:oracle:thin:@%s", tnsName); } public static String getMapperJdbcUrlPropertyName(int mapperId, org.apache.hadoop.conf.Configuration conf) { return String.format("oraoop.mapper.jdbc.url.%d", mapperId); } public static final String SQOOP_JOB_TYPE = "oraoop.sqoop.job.type"; public static void rememberSqoopJobType(OraOopConstants.Sqoop.Tool jobType, org.apache.hadoop.conf.Configuration conf) { conf.set(SQOOP_JOB_TYPE, jobType.name()); } public static OraOopConstants.Sqoop.Tool recallSqoopJobType(org.apache.hadoop.conf.Configuration conf) { String jobType = conf.get(SQOOP_JOB_TYPE); if (jobType == null || jobType.isEmpty()) { throw new RuntimeException( "RecallSqoopJobType() cannot be called unless RememberSqoopJobType() " + "has been used."); } OraOopConstants.Sqoop.Tool result = OraOopConstants.Sqoop.Tool.valueOf(jobType); return result; } public static boolean omitLobAndLongColumnsDuringImport(org.apache.hadoop.conf.Configuration conf) { return conf.getBoolean(OraOopConstants.ORAOOP_IMPORT_OMIT_LOBS_AND_LONG, false); } public static boolean oracleSessionHasBeenKilled(Exception exception) { Throwable ex = exception; while (ex != null) { if (ex instanceof SQLException && ((SQLException) ex).getErrorCode() == 28) { // ORA-00028: your // session has been // killed return true; } ex = ex.getCause(); } return false; } private static String formatTimestampForOracleObjectName(Object oracleDateTime) { // NOTE: Update decodeTimestampFromOracleObjectName() if you modify this // method. String jobTimeStr = OraOopOracleQueries.oraDATEToString(oracleDateTime, OraOopConstants.ORACLE_OBJECT_NAME_DATE_TO_STRING_FORMAT_STRING); return jobTimeStr; // E.g. 20101028_151000 (15 characters) } private static Object decodeTimestampFromOracleObjectName(String oracleObjectNameTimestampFragment) { String dateString = oracleObjectNameTimestampFragment; String dateFormatString = OraOopConstants.ORACLE_OBJECT_NAME_DATE_TO_STRING_FORMAT_STRING; // return oracle.sql.DATE.fromText(oracleObjectNameTimestampFragment // ,OraOopConstants.ORACLE_OBJECT_NAME_DATE_TO_STRING_FORMAT_STRING // ,null); /* * Unfortunately, we don't seem to be able to reliably decode strings into * DATE objects using Oracle. For example, the following string will cause * Oracle to throw an "Invalid Oracle date" exception, due to the time * portion starting with a zero... * oracle.sql.DATE.fromText("20101123 091554", "yyyymmdd hh24miss", null); * * Therefore, we need to manually deconstruct the date string and insert * some colons into the time so that Oracle can decode the string. (This is * therefore an Oracle bug we're working around.) */ try { String year = oracleObjectNameTimestampFragment.substring(0, 4); String month = oracleObjectNameTimestampFragment.substring(4, 6); String day = oracleObjectNameTimestampFragment.substring(6, 8); String hour = oracleObjectNameTimestampFragment.substring(9, 11); String minute = oracleObjectNameTimestampFragment.substring(11, 13); String second = oracleObjectNameTimestampFragment.substring(13, 15); dateString = String.format("%s/%s/%s %s:%s:%s", year, month, day, hour, minute, second); dateFormatString = "yyyy/mm/dd hh24:mi:ss"; return OraOopOracleQueries.oraDATEFromString(dateString, dateFormatString); } catch (Exception ex) { LOG.debug(String.format( "%s could not convert the string \"%s\" into a DATE via the format " + "string \"%s\".\n" + "The error encountered was:\n%s", getCurrentMethodName(), dateString, dateFormatString, getFullExceptionMessage(ex))); return null; } } public static String createExportTablePartitionNameFromOracleTimestamp(Object oracleDateTime) { // Partition name can be up to 30 characters long and must start with a // letter... return OraOopConstants.EXPORT_TABLE_PARTITION_NAME_PREFIX + formatTimestampForOracleObjectName(oracleDateTime); // E.g. ORAOOP_20101028_151000 (22 characters) } public static String createExportTableNamePrefixFromOracleTimestamp(Object oracleDateTime) { // NOTE: Alter decodeExportTableNamePrefix() if you modify this method. // Table name can be 30 characters long and must start with a letter... return OraOopConstants.EXPORT_MAPPER_TABLE_NAME_PREFIX + formatTimestampForOracleObjectName(oracleDateTime); // G1.ORAOOP_20101028_152500 (22 characters) (This is just the prefix, // append "_3" for mapper 4) } public static Object decodeExportTableNamePrefix(String tableNamePrefix) { if (tableNamePrefix == null || tableNamePrefix.isEmpty()) { return null; } if (!tableNamePrefix.startsWith(OraOopConstants.EXPORT_MAPPER_TABLE_NAME_PREFIX)) { return null; } String formattedTimestamp = tableNamePrefix .substring(OraOopConstants.EXPORT_MAPPER_TABLE_NAME_PREFIX.length(), tableNamePrefix.length()); return decodeTimestampFromOracleObjectName(formattedTimestamp); } private static boolean userWantsToCreateExportTableFromTemplate(org.apache.hadoop.conf.Configuration conf) { String exportTableTemplate = conf.get(OraOopConstants.ORAOOP_EXPORT_CREATE_TABLE_TEMPLATE, ""); if (exportTableTemplate.isEmpty()) { return false; } OraOopConstants.Sqoop.Tool tool = OraOopUtilities.recallSqoopJobType(conf); switch (tool) { case UNKNOWN: case EXPORT: return true; default: return false; } } public static boolean enableOracleParallelProcessingDuringExport(org.apache.hadoop.conf.Configuration conf) { return conf.getBoolean(OraOopConstants.ORAOOP_EXPORT_PARALLEL, false); } public static boolean userWantsToCreatePartitionedExportTableFromTemplate( org.apache.hadoop.conf.Configuration conf) { return userWantsToCreateExportTableFromTemplate(conf) && conf.getBoolean(OraOopConstants.ORAOOP_EXPORT_CREATE_TABLE_PARTITIONED, false); } public static boolean userWantsToCreateNonPartitionedExportTableFromTemplate( org.apache.hadoop.conf.Configuration conf) { return userWantsToCreateExportTableFromTemplate(conf) && !conf.getBoolean(OraOopConstants.ORAOOP_EXPORT_CREATE_TABLE_PARTITIONED, false); } public static String generateExportTableSubPartitionName(int mapperId, Object sysDateTime, org.apache.hadoop.conf.Configuration conf) { String partitionName = createExportTablePartitionNameFromOracleTimestamp(sysDateTime); String subPartitionName = String.format("%s_MAP_%d" // <- Should allow for // 1,000 mappers before // exceeding 30 // characters , partitionName // <- Partition name is 22 characters , mapperId); // Check the length of the name... if (subPartitionName.length() > OraOopConstants.Oracle.MAX_IDENTIFIER_LENGTH) { throw new RuntimeException(String.format( "The generated Oracle subpartition name \"%s\" is longer " + "than %d characters.", subPartitionName, OraOopConstants.Oracle.MAX_IDENTIFIER_LENGTH)); } return subPartitionName; } public static String[] generateExportTableSubPartitionNames(int numMappers, Object sysDateTime, org.apache.hadoop.conf.Configuration conf) { String[] result = new String[numMappers]; for (int idx = 0; idx < numMappers; idx++) { result[idx] = generateExportTableSubPartitionName(idx, sysDateTime, conf); } return result; } public static OracleTable generateExportTableMapperTableName(int mapperId, Object sysDateTime, String schema) { //mapperId: should allow 10,000,000 mappers before it exceeds 30 characters. return generateExportTableMapperTableName(Integer.toString(mapperId), sysDateTime, schema); } public static OracleTable generateExportTableMapperTableName(String mapperSuffix, Object sysDateTime, String schema) { // NOTE: Update decodeExportTableMapperTableName() if you alter this method. // Generate a (22 character) prefix to use for the N tables that need to be // created for the N mappers to insert into... String mapperTableNamePrefix = createExportTableNamePrefixFromOracleTimestamp(sysDateTime); // Generate the name... String tableName = String.format("%s_%s", mapperTableNamePrefix // <- 22 // chars , mapperSuffix); // Check the length of the name... if (tableName.length() > OraOopConstants.Oracle.MAX_IDENTIFIER_LENGTH) { throw new RuntimeException( String.format("The generated Oracle table name \"%s\" is longer than " + "%d characters.", tableName, OraOopConstants.Oracle.MAX_IDENTIFIER_LENGTH)); } return new OracleTable(schema, tableName); } /** * Class to wrap the table name to be used for the mapper. */ public static class DecodedExportMapperTableName { private String mapperId; // <- This is not an int, because it might be "CHG" // in the case of a "changes-table". private Object tableDateTime; public String getMapperId() { return mapperId; } public void setMapperId(String newMapperId) { this.mapperId = newMapperId; } public Object getTableDateTime() { return tableDateTime; } public void setTableDateTime(Object newTableDateTime) { this.tableDateTime = newTableDateTime; } } public static DecodedExportMapperTableName decodeExportTableMapperTableName(OracleTable oracleTable) { DecodedExportMapperTableName result = null; try { int lastUnderScoreIndex = oracleTable.getName().lastIndexOf("_"); if (lastUnderScoreIndex == -1) { return result; } String dateFragment = oracleTable.getName().substring(0, lastUnderScoreIndex); String mapperIdFragment = oracleTable.getName().substring(lastUnderScoreIndex + 1, oracleTable.getName().length()); Object sysDateTime = decodeExportTableNamePrefix(dateFragment); if (sysDateTime != null) { result = new DecodedExportMapperTableName(); result.setTableDateTime(sysDateTime); result.setMapperId(mapperIdFragment); } } catch (Exception ex) { LOG.debug(String.format("Error when attempting to decode the export mapper-table name \"%s\".", oracleTable.toString()), ex); } return result; } public static void rememberOracleDateTime(org.apache.hadoop.conf.Configuration conf, String propertyName, String dateTime) { conf.set(propertyName, dateTime); } public static Object recallOracleDateTime(org.apache.hadoop.conf.Configuration conf, String propertyName) { String dateTimeStr = conf.get(propertyName); if (dateTimeStr == null || dateTimeStr.isEmpty()) { throw new RuntimeException( String.format("Unable to recall the value of the property \"%s\".", propertyName)); } return OraOopOracleQueries.oraDATEFromString(dateTimeStr, "yyyy-mm-dd hh24:mi:ss"); } public static UpdateMode getExportUpdateMode(org.apache.hadoop.conf.Configuration conf) { // NB: The Sqoop code does not add the column specified in the // "--update-key" argument // as a configuration property value until quite late in the process. i.e. // After the // OraOopManagerFactory.accept() have been called. // (It is available via sqoopOptions.getUpdateKeyCol() however.) // Therefore, when calling this method you need to be confident that the // export being // performed is actually an "update" export and not an "import" one. // String updateKeyCol = // conf.get(ExportJobBase.SQOOP_EXPORT_UPDATE_COL_KEY); // if(updateKeyCol == null || // updateKeyCol.isEmpty()) // throw new // RuntimeException(String.format("This job is not an update-export. "+ // "i.e. %s has not been specified." // ,ExportJobBase.SQOOP_EXPORT_UPDATE_COL_KEY)); UpdateMode updateMode = UpdateMode.Update; boolean mergeData = conf.getBoolean(OraOopConstants.ORAOOP_EXPORT_MERGE, false); if (mergeData) { updateMode = UpdateMode.Merge; } return updateMode; } public static InsertMode getExportInsertMode(org.apache.hadoop.conf.Configuration conf) { InsertMode result = InsertMode.DirectInsert; if (OraOopUtilities.userWantsToCreatePartitionedExportTableFromTemplate(conf) || conf.getBoolean(OraOopConstants.EXPORT_TABLE_HAS_ORAOOP_PARTITIONS, false)) { result = InsertMode.ExchangePartition; } return result; } public static String getJavaClassPath() { return System.getProperty("java.class.path"); } public static String replaceAll(String inputString, String textToReplace, String replaceWith) { String result = inputString.replaceAll(textToReplace, replaceWith); if (!result.equals(inputString)) { result = replaceAll(result, textToReplace, replaceWith); } return result; } public static String getTemporaryTableStorageClause(org.apache.hadoop.conf.Configuration conf) { String result = conf.get(OraOopConstants.ORAOOP_TEMPORARY_TABLE_STORAGE_CLAUSE, ""); if (result == null) { result = ""; } return result; } public static String getExportTableStorageClause(org.apache.hadoop.conf.Configuration conf) { String result = conf.get(OraOopConstants.ORAOOP_EXPORT_TABLE_STORAGE_CLAUSE, ""); if (result == null) { result = ""; } return result; } public static String[] getExportUpdateKeyColumnNames(SqoopOptions options) { String updateKey = options.getUpdateKeyCol(); return getExtraExportUpdateKeyColumnNames(updateKey, options.getConf()); } public static String[] getExportUpdateKeyColumnNames(org.apache.hadoop.conf.Configuration conf) { String updateKey = conf.get(ExportJobBase.SQOOP_EXPORT_UPDATE_COL_KEY); return getExtraExportUpdateKeyColumnNames(updateKey, conf); } /** * Splits a string separated by commas - the elements can be optionally * enclosed in quotes - this allows the elements to have commas in them. * * @param value * The String to be split * @return A list of values */ public static List<String> splitStringList(String value) { List<String> result = new ArrayList<String>(); if (value != null && !value.isEmpty()) { Pattern pattern = Pattern.compile("([^\",]*|\"[^\"]*\")(,|$)"); Matcher matcher = pattern.matcher(value); while (matcher.find()) { if (matcher.group(1) != null && !matcher.group(1).isEmpty()) { result.add(matcher.group(1)); } } } return result; } /** * Splits a string list separated by commas. If the element is not surrounded * by quotes it will be return in upper case. If the element is enclosed in * quotes it will be returned in the same case and special characters will be * retained. * * @param value * The String to be split * @return A list of values */ public static List<String> splitOracleStringList(String value) { List<String> result = new ArrayList<String>(); List<String> splitValue = splitStringList(value); Pattern pattern = Pattern.compile("(\")([^\"]*)(\")"); for (String element : splitValue) { Matcher matcher = pattern.matcher(element); if (matcher.find()) { result.add(matcher.group(2)); } else { result.add(element.toUpperCase()); } } return result; } private static String[] getExtraExportUpdateKeyColumnNames(String updateKey, org.apache.hadoop.conf.Configuration conf) { if (updateKey == null) { // This must be an "insert-export" if no --update-key has been specified! return new String[0]; } String extraKeys = conf.get(OraOopConstants.ORAOOP_UPDATE_KEY_EXTRA_COLUMNS, ""); if (!extraKeys.isEmpty()) { updateKey += "," + extraKeys; } String[] columnNames = updateKey.split(","); for (int idx = 0; idx < columnNames.length; idx++) { columnNames[idx] = columnNames[idx].trim(); if (!columnNames[idx].startsWith("\"")) { columnNames[idx] = columnNames[idx].toUpperCase(); } } return columnNames; } public static OraOopConstants.AppendValuesHintUsage getOracleAppendValuesHintUsage( org.apache.hadoop.conf.Configuration conf) { if (conf == null) { throw new IllegalArgumentException("The conf argument cannot be null"); } String strUsage = conf.get(OraOopConstants.ORAOOP_ORACLE_APPEND_VALUES_HINT_USAGE); if (strUsage == null) { return OraOopConstants.AppendValuesHintUsage.AUTO; } OraOopConstants.AppendValuesHintUsage result; try { strUsage = strUsage.toUpperCase().trim(); result = OraOopConstants.AppendValuesHintUsage.valueOf(strUsage); } catch (IllegalArgumentException ex) { result = OraOopConstants.AppendValuesHintUsage.AUTO; String errorMsg = String.format( "An invalid value of \"%s\" was specified for the \"%s\" " + "configuration property value.\n" + "\tValid values are: %s\n" + "\tThe default value of %s will be used.", strUsage, OraOopConstants.ORAOOP_ORACLE_APPEND_VALUES_HINT_USAGE, getOraOopOracleAppendValuesHintUsageValues(), OraOopConstants.AppendValuesHintUsage.AUTO.name()); LOG.error(errorMsg); } return result; } private static String getOraOopOracleAppendValuesHintUsageValues() { OraOopConstants.AppendValuesHintUsage[] values = OraOopConstants.AppendValuesHintUsage.values(); StringBuilder result = new StringBuilder((2 * values.length) - 1); // <- // Include // capacity // for // commas for (int idx = 0; idx < values.length; idx++) { OraOopConstants.AppendValuesHintUsage value = values[idx]; if (idx > 0) { result.append(" or "); } result.append(value.name()); } return result.toString(); } public static String getImportHint(org.apache.hadoop.conf.Configuration conf) { String result = null; result = conf.get(OraOopConstants.IMPORT_QUERY_HINT); if (result == null || result.trim().isEmpty()) { result = ""; } else { result = String.format(OraOopConstants.Oracle.HINT_SYNTAX, result); } return result; } public static void appendJavaSecurityEgd(Configuration conf) { String mapredJavaOpts = conf.get("mapred.child.java.opts"); if (mapredJavaOpts == null || !mapredJavaOpts.contains("-Djava.security.egd")) { StringBuilder newMapredJavaOpts = new StringBuilder("-Djava.security.egd=file:///dev/urandom"); if (mapredJavaOpts != null && !mapredJavaOpts.isEmpty()) { newMapredJavaOpts.append(" ").append(mapredJavaOpts); } String newMapredJavaOptsString = newMapredJavaOpts.toString(); conf.set("mapred.child.java.opts", newMapredJavaOptsString); LOG.debug("Updated mapred.child.java.opts from \"" + mapredJavaOpts + "\" to \"" + newMapredJavaOptsString + "\""); } } public static void checkJavaSecurityEgd() { String javaSecurityEgd = System.getProperty("java.security.egd"); if (!"file:///dev/urandom".equals(javaSecurityEgd)) { LOG.warn("System property java.security.egd is not set to " + "file:///dev/urandom - Oracle connections may time out."); } } public static void setCurrentSessionUser(String user) { currentSessionUser = user; } }