Java tutorial
// ============================================================================ // // Copyright (c) 2006-2013, Talend Inc. // // This source code has been automatically generated by_Talend Open Studio for Data Integration // / Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bio2rdf.upload_all_datasets_0_1; import routines.Mathematical; import routines.DataOperation; import routines.BioUtil; import routines.Relational; import routines.TalendDate; import routines.TalendDataGenerator; import routines.Numeric; import routines.MD5; import routines.TalendString; import routines.StringHandling; import routines.system.*; import routines.system.api.*; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import java.math.BigDecimal; import java.io.ByteArrayOutputStream; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.ObjectOutputStream; import java.io.ObjectInputStream; import java.io.IOException; import java.util.Comparator; //the import part of tJava_1 //import java.util.List; @SuppressWarnings("unused") /** * Job: upload_all_datasets Purpose: <br> * Description: <br> * @author test@talend.com * @version 5.5.0.r117820 * @status */ public class upload_all_datasets implements TalendJob { public final Object obj = new Object(); // for transmiting parameters purpose private Object valueObject = null; public Object getValueObject() { return this.valueObject; } public void setValueObject(Object valueObject) { this.valueObject = valueObject; } private final static String defaultCharset = java.nio.charset.Charset.defaultCharset().name(); private final static String utf8Charset = "UTF-8"; // create and load default properties private java.util.Properties defaultProps = new java.util.Properties(); // create application properties with default public class ContextProperties extends java.util.Properties { private static final long serialVersionUID = 1L; public ContextProperties(java.util.Properties properties) { super(properties); } public ContextProperties() { super(); } public void synchronizeContext() { if (workspace != null) { this.setProperty("workspace", workspace.toString()); } if (authorizationKey != null) { this.setProperty("authorizationKey", authorizationKey.toString()); } } public String workspace; public String getWorkspace() { return this.workspace; } public String authorizationKey; public String getAuthorizationKey() { return this.authorizationKey; } } private ContextProperties context = new ContextProperties(); public ContextProperties getContext() { return this.context; } private final String jobVersion = "0.1"; private final String jobName = "upload_all_datasets"; private final String projectName = "BIO2RDF"; public Integer errorCode = null; private String currentComponent = ""; private final java.util.Map<String, Object> globalMap = new java.util.HashMap<String, Object>(); private final java.util.Map<String, Long> start_Hash = new java.util.HashMap<String, Long>(); private final java.util.Map<String, Long> end_Hash = new java.util.HashMap<String, Long>(); private final java.util.Map<String, Boolean> ok_Hash = new java.util.HashMap<String, Boolean>(); public final java.util.List<String[]> globalBuffer = new java.util.ArrayList<String[]>(); public boolean isExportedAsOSGI = false; // OSGi DataSource private final static String KEY_DB_DATASOURCES = "KEY_DB_DATASOURCES"; public void setDataSources(java.util.Map<String, javax.sql.DataSource> dataSources) { java.util.Map<String, routines.system.TalendDataSource> talendDataSources = new java.util.HashMap<String, routines.system.TalendDataSource>(); for (java.util.Map.Entry<String, javax.sql.DataSource> dataSourceEntry : dataSources.entrySet()) { talendDataSources.put(dataSourceEntry.getKey(), new routines.system.TalendDataSource(dataSourceEntry.getValue())); } globalMap.put(KEY_DB_DATASOURCES, talendDataSources); } private final java.io.ByteArrayOutputStream baos = new java.io.ByteArrayOutputStream(); private final java.io.PrintStream errorMessagePS = new java.io.PrintStream( new java.io.BufferedOutputStream(baos)); public String getExceptionStackTrace() { if ("failure".equals(this.getStatus())) { errorMessagePS.flush(); return baos.toString(); } return null; } private java.lang.Exception exception = null; public java.lang.Exception getException() { if ("failure".equals(this.getStatus())) { return this.exception; } return null; } private class TalendException extends java.lang.Exception { private static final long serialVersionUID = 1L; private java.util.Map<String, Object> globalMap = null; private java.lang.Exception e = null; private String currentComponent = null; private String virtualComponentName = null; public void setVirtualComponentName(String virtualComponentName) { this.virtualComponentName = virtualComponentName; } private TalendException(java.lang.Exception e, String errorComponent, final java.util.Map<String, Object> globalMap) { this.currentComponent = errorComponent; this.globalMap = globalMap; this.e = e; } public java.lang.Exception getException() { return this.e; } public String getCurrentComponent() { return this.currentComponent; } public String getExceptionCauseMessage(java.lang.Exception e) { Throwable cause = e; String message = null; int i = 10; while (null != cause && 0 < i--) { message = cause.getMessage(); if (null == message) { cause = cause.getCause(); } else { break; } } if (null == message) { message = e.getClass().getName(); } return message; } @Override public void printStackTrace() { if (!(e instanceof TalendException || e instanceof TDieException)) { if (virtualComponentName != null && currentComponent.indexOf(virtualComponentName + "_") == 0) { globalMap.put(virtualComponentName + "_ERROR_MESSAGE", getExceptionCauseMessage(e)); } globalMap.put(currentComponent + "_ERROR_MESSAGE", getExceptionCauseMessage(e)); System.err.println("Exception in component " + currentComponent); } if (!(e instanceof TDieException)) { if (e instanceof TalendException) { e.printStackTrace(); } else { e.printStackTrace(); e.printStackTrace(errorMessagePS); upload_all_datasets.this.exception = e; } } if (!(e instanceof TalendException)) { try { for (java.lang.reflect.Method m : this.getClass().getEnclosingClass().getMethods()) { if (m.getName().compareTo(currentComponent + "_error") == 0) { m.invoke(upload_all_datasets.this, new Object[] { e, currentComponent, globalMap }); break; } } if (!(e instanceof TDieException)) { } } catch (java.lang.SecurityException e) { this.e.printStackTrace(); } catch (java.lang.IllegalArgumentException e) { this.e.printStackTrace(); } catch (java.lang.IllegalAccessException e) { this.e.printStackTrace(); } catch (java.lang.reflect.InvocationTargetException e) { this.e.printStackTrace(); } } } } public void tFileList_1_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_1_onSubJobError(exception, errorComponent, globalMap); } public void tHttpRequest_1_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_1_onSubJobError(exception, errorComponent, globalMap); } public void tLogRow_2_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_1_onSubJobError(exception, errorComponent, globalMap); } public void tFileList_3_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_3_onSubJobError(exception, errorComponent, globalMap); } public void tHttpRequest_4_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_3_onSubJobError(exception, errorComponent, globalMap); } public void tReplicate_1_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_3_onSubJobError(exception, errorComponent, globalMap); } public void tExtractJSONFields_3_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_3_onSubJobError(exception, errorComponent, globalMap); } public void tLogRow_4_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tFileList_3_onSubJobError(exception, errorComponent, globalMap); } public void tPrejob_1_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tPrejob_1_onSubJobError(exception, errorComponent, globalMap); } public void tJava_1_error(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { end_Hash.put(errorComponent, System.currentTimeMillis()); status = "failure"; tJava_1_onSubJobError(exception, errorComponent, globalMap); } public void tFileList_1_onSubJobError(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), ""); } public void tFileList_3_onSubJobError(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), ""); } public void tPrejob_1_onSubJobError(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), ""); } public void tJava_1_onSubJobError(java.lang.Exception exception, String errorComponent, final java.util.Map<String, Object> globalMap) throws TalendException { resumeUtil.addLog("SYSTEM_LOG", "NODE:" + errorComponent, "", Thread.currentThread().getId() + "", "FATAL", "", exception.getMessage(), ResumeUtil.getExceptionStackTrace(exception), ""); } public static class row1Struct implements routines.system.IPersistableRow<row1Struct> { final static byte[] commonByteArrayLock_BIO2RDF_upload_all_datasets = new byte[0]; static byte[] commonByteArray_BIO2RDF_upload_all_datasets = new byte[0]; public String ResponseContent; public String getResponseContent() { return this.ResponseContent; } private String readString(ObjectInputStream dis) throws IOException { String strReturn = null; int length = 0; length = dis.readInt(); if (length == -1) { strReturn = null; } else { if (length > commonByteArray_BIO2RDF_upload_all_datasets.length) { if (length < 1024 && commonByteArray_BIO2RDF_upload_all_datasets.length == 0) { commonByteArray_BIO2RDF_upload_all_datasets = new byte[1024]; } else { commonByteArray_BIO2RDF_upload_all_datasets = new byte[2 * length]; } } dis.readFully(commonByteArray_BIO2RDF_upload_all_datasets, 0, length); strReturn = new String(commonByteArray_BIO2RDF_upload_all_datasets, 0, length, utf8Charset); } return strReturn; } private void writeString(String str, ObjectOutputStream dos) throws IOException { if (str == null) { dos.writeInt(-1); } else { byte[] byteArray = str.getBytes(utf8Charset); dos.writeInt(byteArray.length); dos.write(byteArray); } } public void readData(ObjectInputStream dis) { synchronized (commonByteArrayLock_BIO2RDF_upload_all_datasets) { try { int length = 0; this.ResponseContent = readString(dis); } catch (IOException e) { throw new RuntimeException(e); } } } public void writeData(ObjectOutputStream dos) { try { // String writeString(this.ResponseContent, dos); } catch (IOException e) { throw new RuntimeException(e); } } public String toString() { StringBuilder sb = new StringBuilder(); sb.append(super.toString()); sb.append("["); sb.append("ResponseContent=" + ResponseContent); sb.append("]"); return sb.toString(); } /** * Compare keys */ public int compareTo(row1Struct other) { int returnValue = -1; return returnValue; } private int checkNullsAndCompare(Object object1, Object object2) { int returnValue = 0; if (object1 instanceof Comparable && object2 instanceof Comparable) { returnValue = ((Comparable) object1).compareTo(object2); } else if (object1 != null && object2 != null) { returnValue = compareStrings(object1.toString(), object2.toString()); } else if (object1 == null && object2 != null) { returnValue = 1; } else if (object1 != null && object2 == null) { returnValue = -1; } else { returnValue = 0; } return returnValue; } private int compareStrings(String string1, String string2) { return string1.compareTo(string2); } } public void tFileList_1Process(final java.util.Map<String, Object> globalMap) throws TalendException { globalMap.put("tFileList_1_SUBPROCESS_STATE", 0); final boolean execStat = this.execStat; String iterateId = ""; int iterateLoop = 0; String currentComponent = ""; java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>(); try { String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName(); boolean resumeIt = currentMethodName.equals(resumeEntryMethodName); if (resumeEntryMethodName == null || resumeIt || globalResumeTicket) {// start // the // resume globalResumeTicket = true; row1Struct row1 = new row1Struct(); /** * [tFileList_1 begin ] start */ int NB_ITERATE_tHttpRequest_1 = 0; // for statistics ok_Hash.put("tFileList_1", false); start_Hash.put("tFileList_1", System.currentTimeMillis()); currentComponent = "tFileList_1"; int tos_count_tFileList_1 = 0; String directory_tFileList_1 = context.workspace; final java.util.List<String> maskList_tFileList_1 = new java.util.ArrayList<String>(); final java.util.List<java.util.regex.Pattern> patternList_tFileList_1 = new java.util.ArrayList<java.util.regex.Pattern>(); maskList_tFileList_1.add("datahub_import_*.json"); for (final String filemask_tFileList_1 : maskList_tFileList_1) { String filemask_compile_tFileList_1 = filemask_tFileList_1; filemask_compile_tFileList_1 = org.apache.oro.text.GlobCompiler.globToPerl5( filemask_tFileList_1.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK); java.util.regex.Pattern fileNamePattern_tFileList_1 = java.util.regex.Pattern .compile(filemask_compile_tFileList_1); patternList_tFileList_1.add(fileNamePattern_tFileList_1); } int NB_FILEtFileList_1 = 0; final boolean case_sensitive_tFileList_1 = true; final java.util.List<java.io.File> list_tFileList_1 = new java.util.ArrayList<java.io.File>(); final java.util.List<String> filePath_tFileList_1 = new java.util.ArrayList<String>(); java.io.File file_tFileList_1 = new java.io.File(directory_tFileList_1); file_tFileList_1.listFiles(new java.io.FilenameFilter() { public boolean accept(java.io.File dir, String name) { java.io.File file = new java.io.File(dir, name); if (!file.isDirectory()) { String fileName_tFileList_1 = file.getName(); for (final java.util.regex.Pattern fileNamePattern_tFileList_1 : patternList_tFileList_1) { if (fileNamePattern_tFileList_1.matcher(fileName_tFileList_1).matches()) { if (!filePath_tFileList_1.contains(file.getAbsolutePath())) { list_tFileList_1.add(file); filePath_tFileList_1.add(file.getAbsolutePath()); } } } } return true; } }); java.util.Collections.sort(list_tFileList_1); for (int i_tFileList_1 = 0; i_tFileList_1 < list_tFileList_1.size(); i_tFileList_1++) { java.io.File files_tFileList_1 = list_tFileList_1.get(i_tFileList_1); String fileName_tFileList_1 = files_tFileList_1.getName(); String currentFileName_tFileList_1 = files_tFileList_1.getName(); String currentFilePath_tFileList_1 = files_tFileList_1.getAbsolutePath(); String currentFileDirectory_tFileList_1 = files_tFileList_1.getParent(); String currentFileExtension_tFileList_1 = null; if (files_tFileList_1.getName().contains(".") && files_tFileList_1.isFile()) { currentFileExtension_tFileList_1 = files_tFileList_1.getName() .substring(files_tFileList_1.getName().lastIndexOf(".") + 1); } else { currentFileExtension_tFileList_1 = ""; } NB_FILEtFileList_1++; globalMap.put("tFileList_1_CURRENT_FILE", currentFileName_tFileList_1); globalMap.put("tFileList_1_CURRENT_FILEPATH", currentFilePath_tFileList_1); globalMap.put("tFileList_1_CURRENT_FILEDIRECTORY", currentFileDirectory_tFileList_1); globalMap.put("tFileList_1_CURRENT_FILEEXTENSION", currentFileExtension_tFileList_1); globalMap.put("tFileList_1_NB_FILE", NB_FILEtFileList_1); /** * [tFileList_1 begin ] stop */ /** * [tFileList_1 main ] start */ currentComponent = "tFileList_1"; tos_count_tFileList_1++; /** * [tFileList_1 main ] stop */ NB_ITERATE_tHttpRequest_1++; iterateLoop++; /** * [tLogRow_2 begin ] start */ ok_Hash.put("tLogRow_2", false); start_Hash.put("tLogRow_2", System.currentTimeMillis()); currentComponent = "tLogRow_2"; int tos_count_tLogRow_2 = 0; // ///////////////////// final String OUTPUT_FIELD_SEPARATOR_tLogRow_2 = "|"; java.io.PrintStream consoleOut_tLogRow_2 = null; StringBuilder strBuffer_tLogRow_2 = null; int nb_line_tLogRow_2 = 0; // ///////////////////// /** * [tLogRow_2 begin ] stop */ /** * [tHttpRequest_1 begin ] start */ ok_Hash.put("tHttpRequest_1", false); start_Hash.put("tHttpRequest_1", System.currentTimeMillis()); currentComponent = "tHttpRequest_1"; int tos_count_tHttpRequest_1 = 0; /** * [tHttpRequest_1 begin ] stop */ /** * [tHttpRequest_1 main ] start */ currentComponent = "tHttpRequest_1"; java.net.URL url_tHttpRequest_1 = new java.net.URL( "http://datahub.io/api/action/package_create"); java.net.HttpURLConnection urlConn_tHttpRequest_1 = (java.net.HttpURLConnection) url_tHttpRequest_1 .openConnection(); urlConn_tHttpRequest_1.setRequestMethod("POST"); urlConn_tHttpRequest_1.setDoOutput(true); urlConn_tHttpRequest_1.setDoInput(true); urlConn_tHttpRequest_1.setUseCaches(false); urlConn_tHttpRequest_1.setRequestProperty("Authorization", context.authorizationKey); urlConn_tHttpRequest_1.connect(); byte[] buffer_tHttpRequest_1 = new byte[1024]; int bos_buffer_tHttpRequest_1 = 0; StringBuilder sb_tHttpRequest_1 = new StringBuilder(); java.io.InputStream bisParam_tHttpRequest_1 = new java.io.BufferedInputStream( new java.io.FileInputStream(((String) globalMap.get("tFileList_1_CURRENT_FILEPATH")))); java.io.OutputStream bosParam_tHttpRequest_1 = new java.io.BufferedOutputStream( urlConn_tHttpRequest_1.getOutputStream()); while ((bos_buffer_tHttpRequest_1 = bisParam_tHttpRequest_1 .read(buffer_tHttpRequest_1)) != -1) { bosParam_tHttpRequest_1.write(buffer_tHttpRequest_1, 0, bos_buffer_tHttpRequest_1); } bosParam_tHttpRequest_1.flush(); bosParam_tHttpRequest_1.close(); bisParam_tHttpRequest_1.close(); if (java.net.HttpURLConnection.HTTP_OK == (urlConn_tHttpRequest_1.getResponseCode())) { java.io.InputStream bis_tHttpRequest_1 = new java.io.BufferedInputStream( urlConn_tHttpRequest_1.getInputStream()); while ((bos_buffer_tHttpRequest_1 = bis_tHttpRequest_1.read(buffer_tHttpRequest_1)) != -1) { sb_tHttpRequest_1 .append(new String(buffer_tHttpRequest_1, 0, bos_buffer_tHttpRequest_1)); } bis_tHttpRequest_1.close(); } else { System.err.println(urlConn_tHttpRequest_1.getResponseCode() + " " + urlConn_tHttpRequest_1.getResponseMessage()); } row1.ResponseContent = sb_tHttpRequest_1.toString(); urlConn_tHttpRequest_1.disconnect(); tos_count_tHttpRequest_1++; /** * [tHttpRequest_1 main ] stop */ /** * [tLogRow_2 main ] start */ currentComponent = "tLogRow_2"; // ///////////////////// strBuffer_tLogRow_2 = new StringBuilder(); if (row1.ResponseContent != null) { // strBuffer_tLogRow_2.append(String.valueOf(row1.ResponseContent)); } // if (globalMap.get("tLogRow_CONSOLE") != null) { consoleOut_tLogRow_2 = (java.io.PrintStream) globalMap.get("tLogRow_CONSOLE"); } else { consoleOut_tLogRow_2 = new java.io.PrintStream( new java.io.BufferedOutputStream(System.out)); globalMap.put("tLogRow_CONSOLE", consoleOut_tLogRow_2); } consoleOut_tLogRow_2.println(strBuffer_tLogRow_2.toString()); consoleOut_tLogRow_2.flush(); nb_line_tLogRow_2++; // //// // //// // ///////////////////// tos_count_tLogRow_2++; /** * [tLogRow_2 main ] stop */ /** * [tHttpRequest_1 end ] start */ currentComponent = "tHttpRequest_1"; ok_Hash.put("tHttpRequest_1", true); end_Hash.put("tHttpRequest_1", System.currentTimeMillis()); /** * [tHttpRequest_1 end ] stop */ /** * [tLogRow_2 end ] start */ currentComponent = "tLogRow_2"; // //// // //// globalMap.put("tLogRow_2_NB_LINE", nb_line_tLogRow_2); // ///////////////////// ok_Hash.put("tLogRow_2", true); end_Hash.put("tLogRow_2", System.currentTimeMillis()); /** * [tLogRow_2 end ] stop */ /** * [tFileList_1 end ] start */ currentComponent = "tFileList_1"; } globalMap.put("tFileList_1_NB_FILE", NB_FILEtFileList_1); ok_Hash.put("tFileList_1", true); end_Hash.put("tFileList_1", System.currentTimeMillis()); /** * [tFileList_1 end ] stop */ } // end the resume if (resumeEntryMethodName == null || globalResumeTicket) { resumeUtil.addLog("CHECKPOINT", "CONNECTION:SUBJOB_OK:tFileList_1:OnSubjobOk", "", Thread.currentThread().getId() + "", "", "", "", "", ""); } tFileList_3Process(globalMap); } catch (java.lang.Exception e) { TalendException te = new TalendException(e, currentComponent, globalMap); throw te; } catch (java.lang.Error error) { throw error; } finally { try { /** * [tFileList_1 finally ] start */ currentComponent = "tFileList_1"; /** * [tFileList_1 finally ] stop */ /** * [tHttpRequest_1 finally ] start */ currentComponent = "tHttpRequest_1"; /** * [tHttpRequest_1 finally ] stop */ /** * [tLogRow_2 finally ] start */ currentComponent = "tLogRow_2"; /** * [tLogRow_2 finally ] stop */ } catch (java.lang.Exception e) { // ignore } catch (java.lang.Error error) { // ignore } resourceMap = null; } globalMap.put("tFileList_1_SUBPROCESS_STATE", 1); } public static class row7Struct implements routines.system.IPersistableRow<row7Struct> { final static byte[] commonByteArrayLock_BIO2RDF_upload_all_datasets = new byte[0]; static byte[] commonByteArray_BIO2RDF_upload_all_datasets = new byte[0]; public String ResponseContent; public String getResponseContent() { return this.ResponseContent; } private String readString(ObjectInputStream dis) throws IOException { String strReturn = null; int length = 0; length = dis.readInt(); if (length == -1) { strReturn = null; } else { if (length > commonByteArray_BIO2RDF_upload_all_datasets.length) { if (length < 1024 && commonByteArray_BIO2RDF_upload_all_datasets.length == 0) { commonByteArray_BIO2RDF_upload_all_datasets = new byte[1024]; } else { commonByteArray_BIO2RDF_upload_all_datasets = new byte[2 * length]; } } dis.readFully(commonByteArray_BIO2RDF_upload_all_datasets, 0, length); strReturn = new String(commonByteArray_BIO2RDF_upload_all_datasets, 0, length, utf8Charset); } return strReturn; } private void writeString(String str, ObjectOutputStream dos) throws IOException { if (str == null) { dos.writeInt(-1); } else { byte[] byteArray = str.getBytes(utf8Charset); dos.writeInt(byteArray.length); dos.write(byteArray); } } public void readData(ObjectInputStream dis) { synchronized (commonByteArrayLock_BIO2RDF_upload_all_datasets) { try { int length = 0; this.ResponseContent = readString(dis); } catch (IOException e) { throw new RuntimeException(e); } } } public void writeData(ObjectOutputStream dos) { try { // String writeString(this.ResponseContent, dos); } catch (IOException e) { throw new RuntimeException(e); } } public String toString() { StringBuilder sb = new StringBuilder(); sb.append(super.toString()); sb.append("["); sb.append("ResponseContent=" + ResponseContent); sb.append("]"); return sb.toString(); } /** * Compare keys */ public int compareTo(row7Struct other) { int returnValue = -1; return returnValue; } private int checkNullsAndCompare(Object object1, Object object2) { int returnValue = 0; if (object1 instanceof Comparable && object2 instanceof Comparable) { returnValue = ((Comparable) object1).compareTo(object2); } else if (object1 != null && object2 != null) { returnValue = compareStrings(object1.toString(), object2.toString()); } else if (object1 == null && object2 != null) { returnValue = 1; } else if (object1 != null && object2 == null) { returnValue = -1; } else { returnValue = 0; } return returnValue; } private int compareStrings(String string1, String string2) { return string1.compareTo(string2); } } public static class row4Struct implements routines.system.IPersistableRow<row4Struct> { final static byte[] commonByteArrayLock_BIO2RDF_upload_all_datasets = new byte[0]; static byte[] commonByteArray_BIO2RDF_upload_all_datasets = new byte[0]; public String ResponseContent; public String getResponseContent() { return this.ResponseContent; } private String readString(ObjectInputStream dis) throws IOException { String strReturn = null; int length = 0; length = dis.readInt(); if (length == -1) { strReturn = null; } else { if (length > commonByteArray_BIO2RDF_upload_all_datasets.length) { if (length < 1024 && commonByteArray_BIO2RDF_upload_all_datasets.length == 0) { commonByteArray_BIO2RDF_upload_all_datasets = new byte[1024]; } else { commonByteArray_BIO2RDF_upload_all_datasets = new byte[2 * length]; } } dis.readFully(commonByteArray_BIO2RDF_upload_all_datasets, 0, length); strReturn = new String(commonByteArray_BIO2RDF_upload_all_datasets, 0, length, utf8Charset); } return strReturn; } private void writeString(String str, ObjectOutputStream dos) throws IOException { if (str == null) { dos.writeInt(-1); } else { byte[] byteArray = str.getBytes(utf8Charset); dos.writeInt(byteArray.length); dos.write(byteArray); } } public void readData(ObjectInputStream dis) { synchronized (commonByteArrayLock_BIO2RDF_upload_all_datasets) { try { int length = 0; this.ResponseContent = readString(dis); } catch (IOException e) { throw new RuntimeException(e); } } } public void writeData(ObjectOutputStream dos) { try { // String writeString(this.ResponseContent, dos); } catch (IOException e) { throw new RuntimeException(e); } } public String toString() { StringBuilder sb = new StringBuilder(); sb.append(super.toString()); sb.append("["); sb.append("ResponseContent=" + ResponseContent); sb.append("]"); return sb.toString(); } /** * Compare keys */ public int compareTo(row4Struct other) { int returnValue = -1; return returnValue; } private int checkNullsAndCompare(Object object1, Object object2) { int returnValue = 0; if (object1 instanceof Comparable && object2 instanceof Comparable) { returnValue = ((Comparable) object1).compareTo(object2); } else if (object1 != null && object2 != null) { returnValue = compareStrings(object1.toString(), object2.toString()); } else if (object1 == null && object2 != null) { returnValue = 1; } else if (object1 != null && object2 == null) { returnValue = -1; } else { returnValue = 0; } return returnValue; } private int compareStrings(String string1, String string2) { return string1.compareTo(string2); } } public static class row6Struct implements routines.system.IPersistableRow<row6Struct> { final static byte[] commonByteArrayLock_BIO2RDF_upload_all_datasets = new byte[0]; static byte[] commonByteArray_BIO2RDF_upload_all_datasets = new byte[0]; public String ResponseContent; public String getResponseContent() { return this.ResponseContent; } private String readString(ObjectInputStream dis) throws IOException { String strReturn = null; int length = 0; length = dis.readInt(); if (length == -1) { strReturn = null; } else { if (length > commonByteArray_BIO2RDF_upload_all_datasets.length) { if (length < 1024 && commonByteArray_BIO2RDF_upload_all_datasets.length == 0) { commonByteArray_BIO2RDF_upload_all_datasets = new byte[1024]; } else { commonByteArray_BIO2RDF_upload_all_datasets = new byte[2 * length]; } } dis.readFully(commonByteArray_BIO2RDF_upload_all_datasets, 0, length); strReturn = new String(commonByteArray_BIO2RDF_upload_all_datasets, 0, length, utf8Charset); } return strReturn; } private void writeString(String str, ObjectOutputStream dos) throws IOException { if (str == null) { dos.writeInt(-1); } else { byte[] byteArray = str.getBytes(utf8Charset); dos.writeInt(byteArray.length); dos.write(byteArray); } } public void readData(ObjectInputStream dis) { synchronized (commonByteArrayLock_BIO2RDF_upload_all_datasets) { try { int length = 0; this.ResponseContent = readString(dis); } catch (IOException e) { throw new RuntimeException(e); } } } public void writeData(ObjectOutputStream dos) { try { // String writeString(this.ResponseContent, dos); } catch (IOException e) { throw new RuntimeException(e); } } public String toString() { StringBuilder sb = new StringBuilder(); sb.append(super.toString()); sb.append("["); sb.append("ResponseContent=" + ResponseContent); sb.append("]"); return sb.toString(); } /** * Compare keys */ public int compareTo(row6Struct other) { int returnValue = -1; return returnValue; } private int checkNullsAndCompare(Object object1, Object object2) { int returnValue = 0; if (object1 instanceof Comparable && object2 instanceof Comparable) { returnValue = ((Comparable) object1).compareTo(object2); } else if (object1 != null && object2 != null) { returnValue = compareStrings(object1.toString(), object2.toString()); } else if (object1 == null && object2 != null) { returnValue = 1; } else if (object1 != null && object2 == null) { returnValue = -1; } else { returnValue = 0; } return returnValue; } private int compareStrings(String string1, String string2) { return string1.compareTo(string2); } } public void tFileList_3Process(final java.util.Map<String, Object> globalMap) throws TalendException { globalMap.put("tFileList_3_SUBPROCESS_STATE", 0); final boolean execStat = this.execStat; String iterateId = ""; int iterateLoop = 0; String currentComponent = ""; java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>(); try { String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName(); boolean resumeIt = currentMethodName.equals(resumeEntryMethodName); if (resumeEntryMethodName == null || resumeIt || globalResumeTicket) {// start // the // resume globalResumeTicket = true; row6Struct row6 = new row6Struct(); row4Struct row4 = new row4Struct(); row7Struct row7 = new row7Struct(); /** * [tFileList_3 begin ] start */ int NB_ITERATE_tHttpRequest_4 = 0; // for statistics ok_Hash.put("tFileList_3", false); start_Hash.put("tFileList_3", System.currentTimeMillis()); currentComponent = "tFileList_3"; int tos_count_tFileList_3 = 0; String directory_tFileList_3 = context.workspace; final java.util.List<String> maskList_tFileList_3 = new java.util.ArrayList<String>(); final java.util.List<java.util.regex.Pattern> patternList_tFileList_3 = new java.util.ArrayList<java.util.regex.Pattern>(); maskList_tFileList_3.add("datahub_import_*.json"); for (final String filemask_tFileList_3 : maskList_tFileList_3) { String filemask_compile_tFileList_3 = filemask_tFileList_3; filemask_compile_tFileList_3 = org.apache.oro.text.GlobCompiler.globToPerl5( filemask_tFileList_3.toCharArray(), org.apache.oro.text.GlobCompiler.DEFAULT_MASK); java.util.regex.Pattern fileNamePattern_tFileList_3 = java.util.regex.Pattern .compile(filemask_compile_tFileList_3); patternList_tFileList_3.add(fileNamePattern_tFileList_3); } int NB_FILEtFileList_3 = 0; final boolean case_sensitive_tFileList_3 = true; final java.util.List<java.io.File> list_tFileList_3 = new java.util.ArrayList<java.io.File>(); final java.util.List<String> filePath_tFileList_3 = new java.util.ArrayList<String>(); java.io.File file_tFileList_3 = new java.io.File(directory_tFileList_3); file_tFileList_3.listFiles(new java.io.FilenameFilter() { public boolean accept(java.io.File dir, String name) { java.io.File file = new java.io.File(dir, name); if (!file.isDirectory()) { String fileName_tFileList_3 = file.getName(); for (final java.util.regex.Pattern fileNamePattern_tFileList_3 : patternList_tFileList_3) { if (fileNamePattern_tFileList_3.matcher(fileName_tFileList_3).matches()) { if (!filePath_tFileList_3.contains(file.getAbsolutePath())) { list_tFileList_3.add(file); filePath_tFileList_3.add(file.getAbsolutePath()); } } } } return true; } }); java.util.Collections.sort(list_tFileList_3); for (int i_tFileList_3 = 0; i_tFileList_3 < list_tFileList_3.size(); i_tFileList_3++) { java.io.File files_tFileList_3 = list_tFileList_3.get(i_tFileList_3); String fileName_tFileList_3 = files_tFileList_3.getName(); String currentFileName_tFileList_3 = files_tFileList_3.getName(); String currentFilePath_tFileList_3 = files_tFileList_3.getAbsolutePath(); String currentFileDirectory_tFileList_3 = files_tFileList_3.getParent(); String currentFileExtension_tFileList_3 = null; if (files_tFileList_3.getName().contains(".") && files_tFileList_3.isFile()) { currentFileExtension_tFileList_3 = files_tFileList_3.getName() .substring(files_tFileList_3.getName().lastIndexOf(".") + 1); } else { currentFileExtension_tFileList_3 = ""; } NB_FILEtFileList_3++; globalMap.put("tFileList_3_CURRENT_FILE", currentFileName_tFileList_3); globalMap.put("tFileList_3_CURRENT_FILEPATH", currentFilePath_tFileList_3); globalMap.put("tFileList_3_CURRENT_FILEDIRECTORY", currentFileDirectory_tFileList_3); globalMap.put("tFileList_3_CURRENT_FILEEXTENSION", currentFileExtension_tFileList_3); globalMap.put("tFileList_3_NB_FILE", NB_FILEtFileList_3); /** * [tFileList_3 begin ] stop */ /** * [tFileList_3 main ] start */ currentComponent = "tFileList_3"; tos_count_tFileList_3++; /** * [tFileList_3 main ] stop */ NB_ITERATE_tHttpRequest_4++; iterateLoop++; /** * [tLogRow_4 begin ] start */ ok_Hash.put("tLogRow_4", false); start_Hash.put("tLogRow_4", System.currentTimeMillis()); currentComponent = "tLogRow_4"; int tos_count_tLogRow_4 = 0; // ///////////////////// final String OUTPUT_FIELD_SEPARATOR_tLogRow_4 = "|"; java.io.PrintStream consoleOut_tLogRow_4 = null; StringBuilder strBuffer_tLogRow_4 = null; int nb_line_tLogRow_4 = 0; // ///////////////////// /** * [tLogRow_4 begin ] stop */ /** * [tExtractJSONFields_3 begin ] start */ ok_Hash.put("tExtractJSONFields_3", false); start_Hash.put("tExtractJSONFields_3", System.currentTimeMillis()); currentComponent = "tExtractJSONFields_3"; int tos_count_tExtractJSONFields_3 = 0; int nb_line_tExtractJSONFields_3 = 0; class ConvertJSONString_tExtractJSONFields_3 { final static int Brace = 0; // { final static int Bracket = 1; // [ private int barceType = -1; private String originalJsonString = ""; private String originalLoopString = ""; private String jsonString4XML = null; private String loopString4XML = null; private String additionRoot = null; public void barceType() { for (int c = 0; c < originalJsonString.length(); ++c) { if (originalJsonString.charAt(c) == '{') { barceType = Brace; break; } else if (originalJsonString.charAt(c) == '[') { barceType = Bracket; break; } } } public void setJsonString(String originalJsonString) { this.originalJsonString = originalJsonString; } public void setLoopString(String originalLoopString) { this.originalLoopString = originalLoopString; } public String getJsonString4XML() { return jsonString4XML; } public String getLoopString4XML() { if (loopString4XML.length() > 1 && loopString4XML.endsWith("/")) { loopString4XML = loopString4XML.substring(0, loopString4XML.length() - 1); } return loopString4XML; } public void setAdditionRoot(String additionRoot) { this.additionRoot = additionRoot; } public String getAdditionRoot() { return additionRoot; } public void generate() { barceType(); jsonString4XML = originalJsonString; loopString4XML = originalLoopString; if (Brace == barceType) { if (isNeedAddRoot(originalJsonString)) { jsonString4XML = "{ \"root\": " + originalJsonString + " }"; loopString4XML = "root" + originalLoopString; setAdditionRoot("root"); } } else if (Bracket == barceType) { jsonString4XML = "{ \"root\" : { \"object\": " + originalJsonString + " } }"; loopString4XML = "root/object" + originalLoopString; setAdditionRoot("object"); } } public boolean isNeedAddRoot(String originalJsonString) { boolean isNeedAddRoot = false; net.sf.json.JSONObject jso = net.sf.json.JSONObject.fromObject(originalJsonString); String jsonKey = ""; Object firstObject = null; if (jso.names().size() == 1) { jsonKey = jso.names().get(0).toString(); firstObject = jso.get(jsonKey); } if (jso.size() > 1 || (firstObject != null && firstObject instanceof net.sf.json.JSONArray && ((net.sf.json.JSONArray) firstObject).size() > 1)) { isNeedAddRoot = true; } return isNeedAddRoot; } } ConvertJSONString_tExtractJSONFields_3 cjs_tExtractJSONFields_3 = new ConvertJSONString_tExtractJSONFields_3(); de.odysseus.staxon.json.JsonXMLConfig config_tExtractJSONFields_3 = new de.odysseus.staxon.json.JsonXMLConfigBuilder() .multiplePI(false).build(); de.odysseus.staxon.json.JsonXMLInputFactory jsonXMLInputFactory_tExtractJSONFields_3 = new de.odysseus.staxon.json.JsonXMLInputFactory( config_tExtractJSONFields_3); javax.xml.stream.XMLOutputFactory xmlOutputFactory_tExtractJSONFields_3 = javax.xml.stream.XMLOutputFactory .newInstance(); boolean isGetWholeJson_tExtractJSONFields_3 = false; class OriginalJSONString_tExtractJSONFields_3 { } OriginalJSONString_tExtractJSONFields_3 originalJSONString_tExtractJSONFields_3 = new OriginalJSONString_tExtractJSONFields_3(); class XML_API_tExtractJSONFields_3 { public boolean isDefNull(org.dom4j.Node node) throws javax.xml.transform.TransformerException { if (node != null && node instanceof org.dom4j.Element) { org.dom4j.Attribute attri = ((org.dom4j.Element) node).attribute("nil"); if (attri != null && ("true").equals(attri.getText())) { return true; } } return false; } public boolean isMissing(org.dom4j.Node node) throws javax.xml.transform.TransformerException { return node == null ? true : false; } public boolean isEmpty(org.dom4j.Node node) throws javax.xml.transform.TransformerException { if (node != null) { return node.getText().length() == 0; } return false; } } String jsonStr_tExtractJSONFields_3 = ""; String xmlStr_tExtractJSONFields_3 = ""; XML_API_tExtractJSONFields_3 xml_api_tExtractJSONFields_3 = new XML_API_tExtractJSONFields_3(); /** * [tExtractJSONFields_3 begin ] stop */ /** * [tReplicate_1 begin ] start */ ok_Hash.put("tReplicate_1", false); start_Hash.put("tReplicate_1", System.currentTimeMillis()); currentComponent = "tReplicate_1"; int tos_count_tReplicate_1 = 0; /** * [tReplicate_1 begin ] stop */ /** * [tHttpRequest_4 begin ] start */ ok_Hash.put("tHttpRequest_4", false); start_Hash.put("tHttpRequest_4", System.currentTimeMillis()); currentComponent = "tHttpRequest_4"; int tos_count_tHttpRequest_4 = 0; /** * [tHttpRequest_4 begin ] stop */ /** * [tHttpRequest_4 main ] start */ currentComponent = "tHttpRequest_4"; java.net.URL url_tHttpRequest_4 = new java.net.URL( "http://datahub.io/api/action/package_update"); java.net.HttpURLConnection urlConn_tHttpRequest_4 = (java.net.HttpURLConnection) url_tHttpRequest_4 .openConnection(); urlConn_tHttpRequest_4.setRequestMethod("POST"); urlConn_tHttpRequest_4.setDoOutput(true); urlConn_tHttpRequest_4.setDoInput(true); urlConn_tHttpRequest_4.setUseCaches(false); urlConn_tHttpRequest_4.setRequestProperty("Authorization", context.authorizationKey); urlConn_tHttpRequest_4.connect(); byte[] buffer_tHttpRequest_4 = new byte[1024]; int bos_buffer_tHttpRequest_4 = 0; StringBuilder sb_tHttpRequest_4 = new StringBuilder(); java.io.InputStream bisParam_tHttpRequest_4 = new java.io.BufferedInputStream( new java.io.FileInputStream(((String) globalMap.get("tFileList_3_CURRENT_FILEPATH")))); java.io.OutputStream bosParam_tHttpRequest_4 = new java.io.BufferedOutputStream( urlConn_tHttpRequest_4.getOutputStream()); while ((bos_buffer_tHttpRequest_4 = bisParam_tHttpRequest_4 .read(buffer_tHttpRequest_4)) != -1) { bosParam_tHttpRequest_4.write(buffer_tHttpRequest_4, 0, bos_buffer_tHttpRequest_4); } bosParam_tHttpRequest_4.flush(); bosParam_tHttpRequest_4.close(); bisParam_tHttpRequest_4.close(); if (java.net.HttpURLConnection.HTTP_OK == (urlConn_tHttpRequest_4.getResponseCode())) { java.io.InputStream bis_tHttpRequest_4 = new java.io.BufferedInputStream( urlConn_tHttpRequest_4.getInputStream()); while ((bos_buffer_tHttpRequest_4 = bis_tHttpRequest_4.read(buffer_tHttpRequest_4)) != -1) { sb_tHttpRequest_4 .append(new String(buffer_tHttpRequest_4, 0, bos_buffer_tHttpRequest_4)); } bis_tHttpRequest_4.close(); } else { System.err.println(urlConn_tHttpRequest_4.getResponseCode() + " " + urlConn_tHttpRequest_4.getResponseMessage()); } row6.ResponseContent = sb_tHttpRequest_4.toString(); urlConn_tHttpRequest_4.disconnect(); tos_count_tHttpRequest_4++; /** * [tHttpRequest_4 main ] stop */ /** * [tReplicate_1 main ] start */ currentComponent = "tReplicate_1"; row4 = new row4Struct(); row4.ResponseContent = row6.ResponseContent; tos_count_tReplicate_1++; /** * [tReplicate_1 main ] stop */ /** * [tExtractJSONFields_3 main ] start */ currentComponent = "tExtractJSONFields_3"; if (row4.ResponseContent != null) {// C_01 jsonStr_tExtractJSONFields_3 = row4.ResponseContent.toString(); row7 = null; org.dom4j.io.SAXReader reader_tExtractJSONFields_3 = new org.dom4j.io.SAXReader(); org.dom4j.Document doc_tExtractJSONFields_3 = null; java.util.HashMap xmlNameSpaceMap_tExtractJSONFields_3 = new java.util.HashMap<String, String>(); org.dom4j.XPath x_tExtractJSONFields_3 = null; java.util.List<org.dom4j.tree.AbstractNode> nodeList_tExtractJSONFields_3 = null; boolean isStructError_tExtractJSONFields_3 = true; String loopQuery_tExtractJSONFields_3 = "/result"; String oraginalJsonStr_tExtractJSONFields_3 = jsonStr_tExtractJSONFields_3; cjs_tExtractJSONFields_3.setJsonString(jsonStr_tExtractJSONFields_3); cjs_tExtractJSONFields_3.setLoopString(loopQuery_tExtractJSONFields_3); java.io.ByteArrayInputStream bais_tExtractJSONFields_3 = null; java.io.ByteArrayOutputStream baos_tExtractJSONFields_3 = new java.io.ByteArrayOutputStream(); try { cjs_tExtractJSONFields_3.generate(); jsonStr_tExtractJSONFields_3 = cjs_tExtractJSONFields_3.getJsonString4XML(); loopQuery_tExtractJSONFields_3 = cjs_tExtractJSONFields_3.getLoopString4XML(); bais_tExtractJSONFields_3 = new ByteArrayInputStream( jsonStr_tExtractJSONFields_3.getBytes("UTF-8")); javax.xml.stream.XMLEventReader xmlEventReader_tExtractJSONFields_3 = jsonXMLInputFactory_tExtractJSONFields_3 .createXMLEventReader(bais_tExtractJSONFields_3); javax.xml.stream.XMLEventWriter xmLEventWriter_tExtractJSONFields_3 = xmlOutputFactory_tExtractJSONFields_3 .createXMLEventWriter(baos_tExtractJSONFields_3, "UTF-8"); xmLEventWriter_tExtractJSONFields_3.add(xmlEventReader_tExtractJSONFields_3); // convert json string to xml xmlStr_tExtractJSONFields_3 = baos_tExtractJSONFields_3.toString(); xmLEventWriter_tExtractJSONFields_3.close(); xmlEventReader_tExtractJSONFields_3.close(); doc_tExtractJSONFields_3 = reader_tExtractJSONFields_3 .read(new java.io.StringReader(xmlStr_tExtractJSONFields_3)); x_tExtractJSONFields_3 = doc_tExtractJSONFields_3 .createXPath(loopQuery_tExtractJSONFields_3); x_tExtractJSONFields_3.setNamespaceURIs(xmlNameSpaceMap_tExtractJSONFields_3); nodeList_tExtractJSONFields_3 = (java.util.List<org.dom4j.tree.AbstractNode>) x_tExtractJSONFields_3 .selectNodes(doc_tExtractJSONFields_3); isStructError_tExtractJSONFields_3 = false; } catch (java.lang.Exception ex_tExtractJSONFields_3) { System.err.println(ex_tExtractJSONFields_3.getMessage()); } finally { baos_tExtractJSONFields_3.close(); if (bais_tExtractJSONFields_3 != null) { bais_tExtractJSONFields_3.close(); } } org.dom4j.Node node_tExtractJSONFields_3 = null; String str_tExtractJSONFields_3 = ""; for (int i_tExtractJSONFields_3 = 0; isStructError_tExtractJSONFields_3 || (nodeList_tExtractJSONFields_3 != null && i_tExtractJSONFields_3 < nodeList_tExtractJSONFields_3 .size()); i_tExtractJSONFields_3++) { if (!isStructError_tExtractJSONFields_3) { row7 = null; row7 = new row7Struct(); org.dom4j.tree.AbstractNode temp_tExtractJSONFields_3 = nodeList_tExtractJSONFields_3 .get(i_tExtractJSONFields_3); nb_line_tExtractJSONFields_3++; try { org.dom4j.XPath xTmp0_tExtractJSONFields_3 = temp_tExtractJSONFields_3 .createXPath("name"); xTmp0_tExtractJSONFields_3 .setNamespaceURIs(xmlNameSpaceMap_tExtractJSONFields_3); Object obj0_tExtractJSONFields_3 = xTmp0_tExtractJSONFields_3 .evaluate(temp_tExtractJSONFields_3); if (obj0_tExtractJSONFields_3 instanceof String || obj0_tExtractJSONFields_3 instanceof Number) { node_tExtractJSONFields_3 = temp_tExtractJSONFields_3; str_tExtractJSONFields_3 = String.valueOf(obj0_tExtractJSONFields_3); } else { node_tExtractJSONFields_3 = xTmp0_tExtractJSONFields_3 .selectSingleNode(temp_tExtractJSONFields_3); str_tExtractJSONFields_3 = xTmp0_tExtractJSONFields_3 .valueOf(temp_tExtractJSONFields_3); } if (xml_api_tExtractJSONFields_3.isDefNull(node_tExtractJSONFields_3)) { row7.ResponseContent = null; } else if (xml_api_tExtractJSONFields_3.isEmpty(node_tExtractJSONFields_3)) { row7.ResponseContent = ""; } else if (xml_api_tExtractJSONFields_3.isMissing(node_tExtractJSONFields_3)) { row7.ResponseContent = null; } else { row7.ResponseContent = str_tExtractJSONFields_3; } } catch (java.lang.Exception ex_tExtractJSONFields_3) { System.err.println(ex_tExtractJSONFields_3.getMessage()); row7 = null; } } isStructError_tExtractJSONFields_3 = false; tos_count_tExtractJSONFields_3++; /** * [tExtractJSONFields_3 main ] stop */ // Start of branch "row7" if (row7 != null) { /** * [tLogRow_4 main ] start */ currentComponent = "tLogRow_4"; // ///////////////////// strBuffer_tLogRow_4 = new StringBuilder(); if (row7.ResponseContent != null) { // strBuffer_tLogRow_4.append(String.valueOf(row7.ResponseContent)); } // if (globalMap.get("tLogRow_CONSOLE") != null) { consoleOut_tLogRow_4 = (java.io.PrintStream) globalMap.get("tLogRow_CONSOLE"); } else { consoleOut_tLogRow_4 = new java.io.PrintStream( new java.io.BufferedOutputStream(System.out)); globalMap.put("tLogRow_CONSOLE", consoleOut_tLogRow_4); } consoleOut_tLogRow_4.println(strBuffer_tLogRow_4.toString()); consoleOut_tLogRow_4.flush(); nb_line_tLogRow_4++; // //// // //// // ///////////////////// tos_count_tLogRow_4++; /** * [tLogRow_4 main ] stop */ } // End of branch "row7" // end for } } // C_01 /** * [tHttpRequest_4 end ] start */ currentComponent = "tHttpRequest_4"; ok_Hash.put("tHttpRequest_4", true); end_Hash.put("tHttpRequest_4", System.currentTimeMillis()); /** * [tHttpRequest_4 end ] stop */ /** * [tReplicate_1 end ] start */ currentComponent = "tReplicate_1"; ok_Hash.put("tReplicate_1", true); end_Hash.put("tReplicate_1", System.currentTimeMillis()); /** * [tReplicate_1 end ] stop */ /** * [tExtractJSONFields_3 end ] start */ currentComponent = "tExtractJSONFields_3"; globalMap.put("tExtractJSONFields_1_NB_LINE", nb_line_tExtractJSONFields_3); ok_Hash.put("tExtractJSONFields_3", true); end_Hash.put("tExtractJSONFields_3", System.currentTimeMillis()); /** * [tExtractJSONFields_3 end ] stop */ /** * [tLogRow_4 end ] start */ currentComponent = "tLogRow_4"; // //// // //// globalMap.put("tLogRow_4_NB_LINE", nb_line_tLogRow_4); // ///////////////////// ok_Hash.put("tLogRow_4", true); end_Hash.put("tLogRow_4", System.currentTimeMillis()); /** * [tLogRow_4 end ] stop */ /** * [tFileList_3 end ] start */ currentComponent = "tFileList_3"; } globalMap.put("tFileList_3_NB_FILE", NB_FILEtFileList_3); ok_Hash.put("tFileList_3", true); end_Hash.put("tFileList_3", System.currentTimeMillis()); /** * [tFileList_3 end ] stop */ } // end the resume } catch (java.lang.Exception e) { TalendException te = new TalendException(e, currentComponent, globalMap); throw te; } catch (java.lang.Error error) { throw error; } finally { try { /** * [tFileList_3 finally ] start */ currentComponent = "tFileList_3"; /** * [tFileList_3 finally ] stop */ /** * [tHttpRequest_4 finally ] start */ currentComponent = "tHttpRequest_4"; /** * [tHttpRequest_4 finally ] stop */ /** * [tReplicate_1 finally ] start */ currentComponent = "tReplicate_1"; /** * [tReplicate_1 finally ] stop */ /** * [tExtractJSONFields_3 finally ] start */ currentComponent = "tExtractJSONFields_3"; /** * [tExtractJSONFields_3 finally ] stop */ /** * [tLogRow_4 finally ] start */ currentComponent = "tLogRow_4"; /** * [tLogRow_4 finally ] stop */ } catch (java.lang.Exception e) { // ignore } catch (java.lang.Error error) { // ignore } resourceMap = null; } globalMap.put("tFileList_3_SUBPROCESS_STATE", 1); } public void tPrejob_1Process(final java.util.Map<String, Object> globalMap) throws TalendException { globalMap.put("tPrejob_1_SUBPROCESS_STATE", 0); final boolean execStat = this.execStat; String iterateId = ""; int iterateLoop = 0; String currentComponent = ""; java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>(); try { String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName(); boolean resumeIt = currentMethodName.equals(resumeEntryMethodName); if (resumeEntryMethodName == null || resumeIt || globalResumeTicket) {// start // the // resume globalResumeTicket = true; /** * [tPrejob_1 begin ] start */ ok_Hash.put("tPrejob_1", false); start_Hash.put("tPrejob_1", System.currentTimeMillis()); currentComponent = "tPrejob_1"; int tos_count_tPrejob_1 = 0; /** * [tPrejob_1 begin ] stop */ /** * [tPrejob_1 main ] start */ currentComponent = "tPrejob_1"; tos_count_tPrejob_1++; /** * [tPrejob_1 main ] stop */ /** * [tPrejob_1 end ] start */ currentComponent = "tPrejob_1"; ok_Hash.put("tPrejob_1", true); end_Hash.put("tPrejob_1", System.currentTimeMillis()); tJava_1Process(globalMap); /** * [tPrejob_1 end ] stop */ } // end the resume } catch (java.lang.Exception e) { TalendException te = new TalendException(e, currentComponent, globalMap); throw te; } catch (java.lang.Error error) { throw error; } finally { try { /** * [tPrejob_1 finally ] start */ currentComponent = "tPrejob_1"; /** * [tPrejob_1 finally ] stop */ } catch (java.lang.Exception e) { // ignore } catch (java.lang.Error error) { // ignore } resourceMap = null; } globalMap.put("tPrejob_1_SUBPROCESS_STATE", 1); } public void tJava_1Process(final java.util.Map<String, Object> globalMap) throws TalendException { globalMap.put("tJava_1_SUBPROCESS_STATE", 0); final boolean execStat = this.execStat; String iterateId = ""; int iterateLoop = 0; String currentComponent = ""; java.util.Map<String, Object> resourceMap = new java.util.HashMap<String, Object>(); try { String currentMethodName = new java.lang.Exception().getStackTrace()[0].getMethodName(); boolean resumeIt = currentMethodName.equals(resumeEntryMethodName); if (resumeEntryMethodName == null || resumeIt || globalResumeTicket) {// start // the // resume globalResumeTicket = true; /** * [tJava_1 begin ] start */ ok_Hash.put("tJava_1", false); start_Hash.put("tJava_1", System.currentTimeMillis()); currentComponent = "tJava_1"; int tos_count_tJava_1 = 0; System.out.println( "'409 Conflict' errors should be displayed: it is normal, the program is trying to create datasets that are already existing (in case they are not) before updating them"); /** * [tJava_1 begin ] stop */ /** * [tJava_1 main ] start */ currentComponent = "tJava_1"; tos_count_tJava_1++; /** * [tJava_1 main ] stop */ /** * [tJava_1 end ] start */ currentComponent = "tJava_1"; ok_Hash.put("tJava_1", true); end_Hash.put("tJava_1", System.currentTimeMillis()); /** * [tJava_1 end ] stop */ } // end the resume } catch (java.lang.Exception e) { TalendException te = new TalendException(e, currentComponent, globalMap); throw te; } catch (java.lang.Error error) { throw error; } finally { try { /** * [tJava_1 finally ] start */ currentComponent = "tJava_1"; /** * [tJava_1 finally ] stop */ } catch (java.lang.Exception e) { // ignore } catch (java.lang.Error error) { // ignore } resourceMap = null; } globalMap.put("tJava_1_SUBPROCESS_STATE", 1); } public String resuming_logs_dir_path = null; public String resuming_checkpoint_path = null; public String parent_part_launcher = null; private String resumeEntryMethodName = null; private boolean globalResumeTicket = false; public boolean watch = false; // portStats is null, it means don't execute the statistics public Integer portStats = null; public int portTraces = 4334; public String clientHost; public String defaultClientHost = "localhost"; public String contextStr = "Default"; public boolean isDefaultContext = true; public String pid = "0"; public String rootPid = null; public String fatherPid = null; public String fatherNode = null; public long startTime = 0; public boolean isChildJob = false; public String log4jLevel = ""; private boolean execStat = true; private ThreadLocal<java.util.Map<String, String>> threadLocal = new ThreadLocal<java.util.Map<String, String>>() { protected java.util.Map<String, String> initialValue() { java.util.Map<String, String> threadRunResultMap = new java.util.HashMap<String, String>(); threadRunResultMap.put("errorCode", null); threadRunResultMap.put("status", ""); return threadRunResultMap; }; }; private java.util.Properties context_param = new java.util.Properties(); public java.util.Map<String, Object> parentContextMap = new java.util.HashMap<String, Object>(); public String status = ""; public static void main(String[] args) { final upload_all_datasets upload_all_datasetsClass = new upload_all_datasets(); int exitCode = upload_all_datasetsClass.runJobInTOS(args); System.exit(exitCode); } public String[][] runJob(String[] args) { int exitCode = runJobInTOS(args); String[][] bufferValue = new String[][] { { Integer.toString(exitCode) } }; return bufferValue; } public boolean hastBufferOutputComponent() { boolean hastBufferOutput = false; return hastBufferOutput; } public int runJobInTOS(String[] args) { // reset status status = ""; String lastStr = ""; for (String arg : args) { if (arg.equalsIgnoreCase("--context_param")) { lastStr = arg; } else if (lastStr.equals("")) { evalParam(arg); } else { evalParam(lastStr + " " + arg); lastStr = ""; } } if (clientHost == null) { clientHost = defaultClientHost; } if (pid == null || "0".equals(pid)) { pid = TalendString.getAsciiRandomString(6); } if (rootPid == null) { rootPid = pid; } if (fatherPid == null) { fatherPid = pid; } else { isChildJob = true; } try { // call job/subjob with an existing context, like: // --context=production. if without this parameter, there will use // the default context instead. java.io.InputStream inContext = upload_all_datasets.class.getClassLoader() .getResourceAsStream("bio2rdf/upload_all_datasets_0_1/contexts/" + contextStr + ".properties"); if (isDefaultContext && inContext == null) { } else { if (inContext != null) { // defaultProps is in order to keep the original context // value defaultProps.load(inContext); inContext.close(); context = new ContextProperties(defaultProps); } else { // print info and job continue to run, for case: // context_param is not empty. System.err.println("Could not find the context " + contextStr); } } if (!context_param.isEmpty()) { context.putAll(context_param); } context.workspace = (String) context.getProperty("workspace"); context.authorizationKey = (String) context.getProperty("authorizationKey"); } catch (java.io.IOException ie) { System.err.println("Could not load context " + contextStr); ie.printStackTrace(); } // get context value from parent directly if (parentContextMap != null && !parentContextMap.isEmpty()) { if (parentContextMap.containsKey("workspace")) { context.workspace = (String) parentContextMap.get("workspace"); } if (parentContextMap.containsKey("authorizationKey")) { context.authorizationKey = (String) parentContextMap.get("authorizationKey"); } } // Resume: init the resumeUtil resumeEntryMethodName = ResumeUtil.getResumeEntryMethodName(resuming_checkpoint_path); resumeUtil = new ResumeUtil(resuming_logs_dir_path, isChildJob, rootPid); resumeUtil.initCommonInfo(pid, rootPid, fatherPid, projectName, jobName, contextStr, jobVersion); // Resume: jobStart resumeUtil.addLog("JOB_STARTED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "", "", "", "", resumeUtil.convertToJsonText(context)); java.util.concurrent.ConcurrentHashMap<Object, Object> concurrentHashMap = new java.util.concurrent.ConcurrentHashMap<Object, Object>(); globalMap.put("concurrentHashMap", concurrentHashMap); long startUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); long endUsedMemory = 0; long end = 0; startTime = System.currentTimeMillis(); this.globalResumeTicket = true;// to run tPreJob try { errorCode = null; tPrejob_1Process(globalMap); if (!"failure".equals(status)) { status = "end"; } } catch (TalendException e_tPrejob_1) { globalMap.put("tPrejob_1_SUBPROCESS_STATE", -1); e_tPrejob_1.printStackTrace(); } this.globalResumeTicket = false;// to run others jobs try { errorCode = null; tFileList_1Process(globalMap); if (!"failure".equals(status)) { status = "end"; } } catch (TalendException e_tFileList_1) { globalMap.put("tFileList_1_SUBPROCESS_STATE", -1); e_tFileList_1.printStackTrace(); } this.globalResumeTicket = true;// to run tPostJob end = System.currentTimeMillis(); if (watch) { System.out.println((end - startTime) + " milliseconds"); } endUsedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); if (false) { System.out.println((endUsedMemory - startUsedMemory) + " bytes memory increase when running : upload_all_datasets"); } int returnCode = 0; if (errorCode == null) { returnCode = status != null && status.equals("failure") ? 1 : 0; } else { returnCode = errorCode.intValue(); } resumeUtil.addLog("JOB_ENDED", "JOB:" + jobName, parent_part_launcher, Thread.currentThread().getId() + "", "", "" + returnCode, "", "", ""); return returnCode; } public void destroy() { } private java.util.Map<String, Object> getSharedConnections4REST() { java.util.Map<String, Object> connections = new java.util.HashMap<String, Object>(); return connections; } private void evalParam(String arg) { if (arg.startsWith("--resuming_logs_dir_path")) { resuming_logs_dir_path = arg.substring(25); } else if (arg.startsWith("--resuming_checkpoint_path")) { resuming_checkpoint_path = arg.substring(27); } else if (arg.startsWith("--parent_part_launcher")) { parent_part_launcher = arg.substring(23); } else if (arg.startsWith("--watch")) { watch = true; } else if (arg.startsWith("--stat_port=")) { String portStatsStr = arg.substring(12); if (portStatsStr != null && !portStatsStr.equals("null")) { portStats = Integer.parseInt(portStatsStr); } } else if (arg.startsWith("--trace_port=")) { portTraces = Integer.parseInt(arg.substring(13)); } else if (arg.startsWith("--client_host=")) { clientHost = arg.substring(14); } else if (arg.startsWith("--context=")) { contextStr = arg.substring(10); isDefaultContext = false; } else if (arg.startsWith("--father_pid=")) { fatherPid = arg.substring(13); } else if (arg.startsWith("--root_pid=")) { rootPid = arg.substring(11); } else if (arg.startsWith("--father_node=")) { fatherNode = arg.substring(14); } else if (arg.startsWith("--pid=")) { pid = arg.substring(6); } else if (arg.startsWith("--context_param")) { String keyValue = arg.substring(16); int index = -1; if (keyValue != null && (index = keyValue.indexOf('=')) > -1) { if (fatherPid == null) { context_param.put(keyValue.substring(0, index), replaceEscapeChars(keyValue.substring(index + 1))); } else { // the subjob won't escape the especial chars context_param.put(keyValue.substring(0, index), keyValue.substring(index + 1)); } } } else if (arg.startsWith("--log4jLevel=")) { log4jLevel = arg.substring(13); } } private final String[][] escapeChars = { { "\\\\", "\\" }, { "\\n", "\n" }, { "\\'", "\'" }, { "\\r", "\r" }, { "\\f", "\f" }, { "\\b", "\b" }, { "\\t", "\t" } }; private String replaceEscapeChars(String keyValue) { if (keyValue == null || ("").equals(keyValue.trim())) { return keyValue; } StringBuilder result = new StringBuilder(); int currIndex = 0; while (currIndex < keyValue.length()) { int index = -1; // judege if the left string includes escape chars for (String[] strArray : escapeChars) { index = keyValue.indexOf(strArray[0], currIndex); if (index >= 0) { result.append(keyValue.substring(currIndex, index + strArray[0].length()).replace(strArray[0], strArray[1])); currIndex = index + strArray[0].length(); break; } } // if the left string doesn't include escape chars, append the left // into the result if (index < 0) { result.append(keyValue.substring(currIndex)); currIndex = currIndex + keyValue.length(); } } return result.toString(); } public Integer getErrorCode() { return errorCode; } public String getStatus() { return status; } ResumeUtil resumeUtil = null; } /************************************************************************************************ * 72987 characters generated by Talend Open Studio for Data Integration on the * 11 aot 2014 16:32:21 EDT ************************************************************************************************/