Java tutorial
package io.frictionlessdata.datapackage; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import java.io.BufferedOutputStream; import java.io.BufferedReader; import java.net.URL; import org.json.*; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.apache.commons.lang3.StringUtils; import org.apache.commons.validator.routines.UrlValidator; import org.everit.json.schema.ValidationException; /** * Load, validate and create a datapackage object. */ public class Package { private static final int JSON_INDENT_FACTOR = 4; private static final String DATAPACKAGE_FILENAME = "datapackage.json"; public static final String JSON_KEY_RESOURCES = "resources"; public static final String JSON_KEY_NAME = "name"; public static final String JSON_KEY_PROFILE = "profile"; private String basePath = null; private JSONObject jsonObject = new JSONObject(); private boolean strictValidation = false; private List<Resource> resources = new ArrayList(); private List<Exception> errors = new ArrayList(); private Validator validator = new Validator(); public Package() { } /** * Load from native Java JSONObject. * @param jsonObjectSource * @param strict * @throws IOException * @throws DataPackageException * @throws ValidationException */ public Package(JSONObject jsonObjectSource, boolean strict) throws IOException, DataPackageException, ValidationException { this.setJson(jsonObjectSource); this.strictValidation = strict; this.validate(); } /** * Load from native Java JSONObject. * @param jsonObjectSource * @throws IOException * @throws DataPackageException */ public Package(JSONObject jsonObjectSource) throws IOException, DataPackageException { this(jsonObjectSource, false); } /** * Load from String representation of JSON object or from a zip file path. * @param jsonStringSource * @param strict * @throws IOException * @throws DataPackageException * @throws ValidationException */ public Package(String jsonStringSource, boolean strict) throws IOException, DataPackageException, ValidationException { this.strictValidation = strict; // If zip file is given. if (jsonStringSource.toLowerCase().endsWith(".zip")) { // Read in memory the file inside the zip. ZipFile zipFile = new ZipFile(jsonStringSource); ZipEntry entry = zipFile.getEntry(DATAPACKAGE_FILENAME); // Throw exception if expected datapackage.json file not found. if (entry == null) { throw new DataPackageException( "The zip file does not contain the expected file: " + DATAPACKAGE_FILENAME); } // Read the datapackage.json file inside the zip try (InputStream is = zipFile.getInputStream(entry)) { StringBuilder out = new StringBuilder(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { String line = null; while ((line = reader.readLine()) != null) { out.append(line); } } // Create and set the JSONObject for the datapackage.json that was read from inside the zip file. this.setJson(new JSONObject(out.toString())); // Validate. this.validate(); } } else { // Create and set the JSONObject fpr the String representation of desriptor JSON object. this.setJson(new JSONObject(jsonStringSource)); // If String representation of desriptor JSON object is provided. this.validate(); } } /** * Load from String representation of JSON object or from a zip file path. * @param jsonStringSource * @throws DataPackageException * @throws ValidationException * @throws IOException */ public Package(String jsonStringSource) throws DataPackageException, ValidationException, IOException { this(jsonStringSource, false); } /** * Load from URL (must be in either 'http' or 'https' schemes). * @param urlSource * @param strict * @throws DataPackageException * @throws ValidationException * @throws IOException * @throws FileNotFoundException */ public Package(URL urlSource, boolean strict) throws DataPackageException, ValidationException, IOException, FileNotFoundException { this.strictValidation = strict; // Get string content of given remove file. String jsonString = getJsonStringContentFromRemoteFile(urlSource); // Create JSONObject and validate. this.setJson(new JSONObject(jsonString)); this.validate(); } /** * Load from URL (must be in either 'http' or 'https' schemes). * No validation by default. * @param urlSource * @throws DataPackageException * @throws IOException * @throws FileNotFoundException */ public Package(URL urlSource) throws DataPackageException, IOException, FileNotFoundException { this(urlSource, false); } /** * Load from local file system path. * @param filePath * @param basePath * @param strict * @throws DataPackageException * @throws ValidationException * @throws FileNotFoundException */ public Package(String filePath, String basePath, boolean strict) throws IOException, DataPackageException, ValidationException, FileNotFoundException { this.strictValidation = strict; File sourceFile = null; if (StringUtils.isEmpty(basePath)) { // There is no basePath, i.e. it is empty ("") or null. // Hence the source is the absolute path of the file. // In this case we grab the directory of the source path and set it as the basePath. sourceFile = new File(filePath); } else { // There is a basePath. Construct the absolute path and load it. String absoluteFilePath = basePath + "/" + filePath; sourceFile = new File(absoluteFilePath); } if (sourceFile.exists()) { // Set base path this.setBasePath(sourceFile.getParent()); // Read file, it should be a JSON. String sourceJsonString = this.getJsonStringContentFromLocalFile(sourceFile.getAbsolutePath()); JSONObject sourceJsonObject = new JSONObject(sourceJsonString); this.setJson(sourceJsonObject); this.validate(); } else { throw new FileNotFoundException(); } } /** * Load from local file system path. * No validation by default. * @param filePath * @param basePath * @throws IOException * @throws DataPackageException * @throws FileNotFoundException */ public Package(String filePath, String basePath) throws IOException, DataPackageException, FileNotFoundException { this(filePath, basePath, false); } public void save(String outputFilePath) throws IOException, DataPackageException { if (outputFilePath.toLowerCase().endsWith(".json")) { this.saveJson(outputFilePath); } else if (outputFilePath.toLowerCase().endsWith(".zip")) { this.saveZip(outputFilePath); } else { throw new DataPackageException("Unrecognized file format."); } } private void saveJson(String outputFilePath) throws IOException, DataPackageException { try (FileWriter file = new FileWriter(outputFilePath)) { file.write(this.getJson().toString(JSON_INDENT_FACTOR)); } } private void saveZip(String outputFilePath) throws IOException, DataPackageException { try (FileOutputStream fos = new FileOutputStream(outputFilePath)) { try (BufferedOutputStream bos = new BufferedOutputStream(fos)) { try (ZipOutputStream zos = new ZipOutputStream(bos)) { // File is not on the disk, test.txt indicates // only the file name to be put into the zip. ZipEntry entry = new ZipEntry("datapackage.json"); zos.putNextEntry(entry); zos.write(this.getJson().toString(JSON_INDENT_FACTOR).getBytes()); zos.closeEntry(); } } } } public void infer() { this.infer(false); } public void infer(boolean pattern) { throw new UnsupportedOperationException(); } public Resource getResource(String resourceName) { Iterator<Resource> iter = this.resources.iterator(); while (iter.hasNext()) { Resource resource = iter.next(); if (resource.getName().equalsIgnoreCase(resourceName)) { return resource; } } return null; } public List<Resource> getResources() { return this.resources; } public void addResource(Resource resource) throws IOException, ValidationException, DataPackageException { // If a name property isn't given... if (resource.getName() == null) { DataPackageException dpe = new DataPackageException("The resource does not have a name property."); if (this.strictValidation) { throw dpe; } else { errors.add(dpe); } } else if (resource.getPath() == null && (resource.getData() == null || resource.getFormat() == null)) { DataPackageException dpe = new DataPackageException( "Invalid Resource. The path property or the data and format properties cannot be null."); if (this.strictValidation) { throw dpe; } else { errors.add(dpe); } } else { Iterator<Resource> iter = this.resources.iterator(); // Check if there is duplication. while (iter.hasNext()) { if (iter.next().getName().equalsIgnoreCase(resource.getName())) { DataPackageException dpe = new DataPackageException( "A resource with the same name already exists."); if (this.strictValidation) { throw dpe; } else { errors.add(dpe); } } } } // Validate. this.validate(); this.resources.add(resource); } public void removeResource(String name) { this.resources.removeIf(resource -> resource.getName().equalsIgnoreCase(name)); } public Object getProperty(String key) { return this.getJson().get(key); } public Object getPropertyString(String key) { return this.getJson().getString(key); } public Object getPropertyJSONObject(String key) { return this.getJson().getJSONObject(key); } public Object getPropertyJSONArray(String key) { return this.getJson().getJSONArray(key); } public void addProperty(String key, String value) throws DataPackageException { if (this.getJson().has(key)) { throw new DataPackageException("A property with the same key already exists."); } else { this.getJson().put(key, value); } } public void addProperty(String key, JSONObject value) throws DataPackageException { if (this.getJson().has(key)) { throw new DataPackageException("A property with the same key already exists."); } else { this.getJson().put(key, value); } } public void addProperty(String key, JSONArray value) throws DataPackageException { if (this.getJson().has(key)) { throw new DataPackageException("A property with the same key already exists."); } else { this.getJson().put(key, value); } } public void removeProperty(String key) { this.getJson().remove(key); } /** * Validation is strict or unstrict depending on how the package was * instanciated with the strict flag. * @throws IOException * @throws DataPackageException * @throws ValidationException */ public final void validate() throws IOException, DataPackageException, ValidationException { try { this.validator.validate(this.getJson()); } catch (ValidationException ve) { if (this.strictValidation) { throw ve; } else { errors.add(ve); } } } final public String getBasePath() { return this.basePath; } final public void setBasePath(String basePath) { this.basePath = basePath; } public JSONObject getJson() { Iterator<Resource> resourceIter = resources.iterator(); JSONArray resourcesJsonArray = new JSONArray(); while (resourceIter.hasNext()) { Resource resource = resourceIter.next(); resourcesJsonArray.put(resource.getJson()); } if (resourcesJsonArray.length() > 0) { this.jsonObject.put(JSON_KEY_RESOURCES, resourcesJsonArray); } return this.jsonObject; } public List<Exception> getErrors() { return this.errors; } private String getJsonStringContentFromRemoteFile(URL url) throws IOException { try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) { StringBuilder builder = new StringBuilder(); int read; char[] chars = new char[1024]; while ((read = reader.read(chars)) != -1) { builder.append(chars, 0, read); } return builder.toString(); } } private String getJsonStringContentFromLocalFile(String absoluteFilePath) throws JSONException { // Read file, it should be a JSON. try { String jsonString = new String(Files.readAllBytes(Paths.get(absoluteFilePath))); return jsonString; } catch (IOException ioe) { // FIXME: Come up with better exception handling? return null; } } private void setJson(JSONObject jsonObjectSource) throws IOException, MalformedURLException, FileNotFoundException, DataPackageException { this.jsonObject = jsonObjectSource; // Create Resource list, is there are resources. if (jsonObjectSource.has(JSON_KEY_RESOURCES)) { JSONArray resourcesJsonArray = jsonObjectSource.getJSONArray(JSON_KEY_RESOURCES); for (int i = 0; i < resourcesJsonArray.length(); i++) { JSONObject resourceJson = resourcesJsonArray.getJSONObject(i); //FIXME: Again, could be greatly simplified amd much more // elegant if we use a library like GJSON... String name = resourceJson.has(Resource.JSON_KEY_NAME) ? resourceJson.getString(Resource.JSON_KEY_NAME) : null; Object path = resourceJson.has(Resource.JSON_KEY_PATH) ? resourceJson.get(Resource.JSON_KEY_PATH) : null; Object data = resourceJson.has(Resource.JSON_KEY_DATA) ? resourceJson.get(Resource.JSON_KEY_DATA) : null; String profile = resourceJson.has(Resource.JSON_KEY_PROFILE) ? resourceJson.getString(Resource.JSON_KEY_PROFILE) : null; String title = resourceJson.has(Resource.JSON_KEY_TITLE) ? resourceJson.getString(Resource.JSON_KEY_TITLE) : null; String description = resourceJson.has(Resource.JSON_KEY_DESCRIPTION) ? resourceJson.getString(Resource.JSON_KEY_DESCRIPTION) : null; String format = resourceJson.has(Resource.JSON_KEY_FORMAT) ? resourceJson.getString(Resource.JSON_KEY_FORMAT) : null; String mediaType = resourceJson.has(Resource.JSON_KEY_MEDIA_TYPE) ? resourceJson.getString(Resource.JSON_KEY_MEDIA_TYPE) : null; String encoding = resourceJson.has(Resource.JSON_KEY_ENCODING) ? resourceJson.getString(Resource.JSON_KEY_ENCODING) : null; Integer bytes = resourceJson.has(Resource.JSON_KEY_BYTES) ? resourceJson.getInt(Resource.JSON_KEY_BYTES) : null; String hash = resourceJson.has(Resource.JSON_KEY_HASH) ? resourceJson.getString(Resource.JSON_KEY_HASH) : null; JSONArray sources = resourceJson.has(Resource.JSON_KEY_SOURCES) ? resourceJson.getJSONArray(Resource.JSON_KEY_SOURCES) : null; JSONArray licenses = resourceJson.has(Resource.JSON_KEY_LICENSES) ? resourceJson.getJSONArray(Resource.JSON_KEY_LICENSES) : null; // Get the schema and dereference it. Enables validation against it. Object schemaObj = resourceJson.has(Resource.JSON_KEY_SCHEMA) ? resourceJson.get(Resource.JSON_KEY_SCHEMA) : null; JSONObject dereferencedSchema = this.getDereferencedObject(schemaObj); // Now we can build the resource objects Resource resource = null; if (path != null) { // Get the dialect and dereference it. Enables validation against it. Object dialectObj = resourceJson.has(Resource.JSON_KEY_DIALECT) ? resourceJson.get(Resource.JSON_KEY_DIALECT) : null; JSONObject dereferencedDialect = this.getDereferencedObject(dialectObj); resource = new Resource(name, path, dereferencedSchema, dereferencedDialect, profile, title, description, mediaType, encoding, bytes, hash, sources, licenses); } else if (data != null && format != null) { resource = new Resource(name, data, format, dereferencedSchema, profile, title, description, mediaType, encoding, bytes, hash, sources, licenses); } else { DataPackageException dpe = new DataPackageException( "Invalid Resource. The path property or the data and format properties cannot be null."); if (this.strictValidation) { this.jsonObject = null; this.resources.clear(); throw dpe; } else { this.errors.add(dpe); } } if (resource != null) { this.resources.add(resource); } } } } private JSONObject getDereferencedObject(Object obj) throws IOException, FileNotFoundException, MalformedURLException { // The JSONObject that will represent the schema. JSONObject dereferencedObj = null; // Object is already a dereferences object. if (obj instanceof JSONObject) { // Don't need to do anything, just cast and return. dereferencedObj = (JSONObject) obj; } else if (obj instanceof String) { // The string value of the given object value. String objStr = (String) obj; // If object value is Url. // Grab the JSON string content of that remote file. String[] schemes = { "http", "https" }; UrlValidator urlValidator = new UrlValidator(schemes); if (urlValidator.isValid(objStr)) { // Create the dereferenced object from the remote file. String jsonContentString = this.getJsonStringContentFromRemoteFile(new URL(objStr)); dereferencedObj = new JSONObject(jsonContentString); } else { // If schema is file path. File sourceFile = new File(objStr); if (sourceFile.exists()) { // Create the dereferenced schema object from the local file. String jsonContentString = this.getJsonStringContentFromLocalFile(sourceFile.getAbsolutePath()); dereferencedObj = new JSONObject(jsonContentString); } else { throw new FileNotFoundException("Local file not found: " + sourceFile); } } } return dereferencedObj; } }