cross.io.xml.FragmentXMLSerializer.java Source code

Java tutorial

Introduction

Here is the source code for cross.io.xml.FragmentXMLSerializer.java

Source

/*
 * Cross, common runtime object support system.
 * Copyright (C) 2008-2012, The authors of Cross. All rights reserved.
 *
 * Project website: http://maltcms.sf.net
 *
 * Cross may be used under the terms of either the
 *
 * GNU Lesser General Public License (LGPL)
 * http://www.gnu.org/licenses/lgpl.html
 *
 * or the
 *
 * Eclipse Public License (EPL)
 * http://www.eclipse.org/org/documents/epl-v10.php
 *
 * As a user/recipient of Cross, you may choose which license to receive the code
 * under. Certain files or entire directories may not be covered by this
 * dual license, but are subject to licenses compatible to both LGPL and EPL.
 * License exceptions are explicitly declared in all relevant files or in a
 * LICENSE file in the relevant directories.
 *
 * Cross is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. Please consult the relevant license documentation
 * for details.
 */
package cross.io.xml;

import cross.datastructures.fragments.FileFragment;
import cross.datastructures.fragments.IFileFragment;
import cross.datastructures.fragments.IFragment;
import cross.datastructures.fragments.IVariableFragment;
import cross.datastructures.fragments.VariableFragment;
import cross.datastructures.tools.EvalTools;
import cross.datastructures.tools.FileTools;
import cross.exception.ResourceNotAvailableException;
import cross.io.IDataSource;
import cross.io.misc.Base64;
import cross.tools.StringTools;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.StreamTokenizer;
import java.io.StringReader;
import java.text.NumberFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.event.ConfigurationEvent;
import org.jdom.Attribute;
import org.jdom.DataConversionException;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
import ucar.ma2.Array;
import ucar.ma2.DataType;
import static ucar.ma2.DataType.INT;
import static ucar.ma2.DataType.LONG;
import ucar.ma2.IndexIterator;
import ucar.ma2.InvalidRangeException;
import ucar.ma2.Range;
import ucar.nc2.Dimension;

/**
 * Serializes a FileFragment and it's children structurally, with array data,
 * but array data is currently experimental.
 *
 * @author Nils Hoffmann
 *
 */
@Slf4j
public class FragmentXMLSerializer implements IDataSource {

    private enum Mode {

        UNDEF, DOUBLE, FLOAT, LONG, INTEGER, BYTE, SHORT, BOOLEAN, OBJECT, STRING
    }

    /*
     * (non-Javadoc)
     *
     * @see
     * cross.io.IDataSource#canRead(cross.datastructures.fragments.IFileFragment
     * )
     */
    @Override
    public int canRead(final IFileFragment ff) {
        if (ff.getUri().getPath().endsWith("maltcms.xml")) {
            return 1;
        }
        return 0;
    }

    /*
     * (non-Javadoc)
     *
     * @see org.apache.commons.configuration.event.ConfigurationListener#
     * configurationChanged
     * (org.apache.commons.configuration.event.ConfigurationEvent)
     */

    /**
     *
     * @param arg0
     */

    @Override
    public void configurationChanged(final ConfigurationEvent arg0) {
        // TODO Auto-generated method stub
    }

    /*
     * (non-Javadoc)
     *
     * @see
     * cross.io.IDataSource#configure(org.apache.commons.configuration.Configuration
     * )
     */
    @Override
    public void configure(final Configuration configuration) {
        // TODO Auto-generated method stub
    }

    /**
     *
     * @param location
     * @return
     * @throws IOException
     */
    public IFileFragment deserialize(final String location) throws IOException {
        final File f = new File(location);
        log.info("Deserializing {}", f.getAbsolutePath());
        final SAXBuilder parser = new SAXBuilder();
        IFileFragment ff = null;
        try {
            final Document doc = parser.build(new BufferedInputStream(new FileInputStream(f)));
            log.debug("{}", doc.toString());
            final Element root = doc.getRootElement();
            ff = handleFile(root);

        } catch (final JDOMException e) {
            throw new IOException(e.fillInStackTrace());
        }
        return ff;
    }

    /**
     *
     * @param f
     * @param group
     */
    protected void handleAttributes(final IFragment f, final Element group) {
        final Element attributes = group.getChild("attributes");
        if (attributes != null) {
            int size = attributes.getChildren("attribute").size();
            try {
                size = attributes.getAttribute("size").getIntValue();
            } catch (final DataConversionException e) {
                log.error(e.getLocalizedMessage());
            }
            final ucar.nc2.Attribute[] attribA = new ucar.nc2.Attribute[size];
            int cnt = 0;
            final List<?> l = attributes.getChildren("attribute");
            for (final Object o : l) {
                final Element attribute = (Element) o;
                final String name = attribute.getAttributeValue("name");
                final String value = attribute.getAttributeValue("value");
                attribA[cnt++] = new ucar.nc2.Attribute(name, value);
            }
            f.setAttributes(attribA);
        }
    }

    /**
     * @param name
     * @param dims
     * @param data
     * @return
     */
    private Array handleData(final String name, final Dimension[] dims, final Element data) {
        return handleData(name, dims, data, null);
    }

    /**
     * @param name
     * @param dims
     * @param data
     * @return
     */
    private Array handleData(final String name, final Dimension[] dims, final Element data, final Range[] ranges) {
        EvalTools.notNull(dims, this);
        final String dec = new String(Base64.decode(data.getText(), Base64.GZIP));
        final StreamTokenizer st = new StreamTokenizer(new StringReader(dec));
        final NumberFormat nf = NumberFormat.getNumberInstance(Locale.US);
        final int[] shape = new int[dims.length];
        int d = 0;
        for (final Dimension dim : dims) {
            shape[d++] = dim.getLength();
        }
        Array a = null;
        IndexIterator idx = null;
        int tok = -1;
        // log.info("DataType of array: {}",
        // dt.getPrimitiveClassType().getName());
        Mode m = Mode.UNDEF;
        Object o = null;
        try {
            while ((tok = st.nextToken()) != StreamTokenizer.TT_EOL) {
                if (tok == StreamTokenizer.TT_WORD) {

                    if (m == Mode.UNDEF) {
                        try {
                            o = nf.parse(st.sval);

                            if (o instanceof Double) {
                                m = Mode.DOUBLE;
                                a = Array.factory(DataType.DOUBLE, shape);
                            } else if (o instanceof Float) {
                                m = Mode.FLOAT;
                                a = Array.factory(DataType.FLOAT, shape);
                            } else if (o instanceof Long) {
                                m = Mode.LONG;
                                a = Array.factory(DataType.LONG, shape);
                            } else if (o instanceof Integer) {
                                m = Mode.INTEGER;
                                a = Array.factory(DataType.INT, shape);
                            } else if (o instanceof Byte) {
                                m = Mode.BYTE;
                                a = Array.factory(DataType.BYTE, shape);
                            } else if (o instanceof Short) {
                                m = Mode.SHORT;
                                a = Array.factory(DataType.SHORT, shape);
                            }
                        } catch (final ParseException pe) {
                            if (st.sval.equalsIgnoreCase("true") || st.sval.equalsIgnoreCase("false")) {
                                m = Mode.BOOLEAN;
                                a = Array.factory(DataType.BOOLEAN, shape);
                            } else {
                                m = Mode.STRING;
                                a = Array.factory(DataType.STRING, shape);
                            }
                        }
                    } else {
                        if (idx == null) {
                            idx = a.getIndexIterator();
                        }
                        switch (m) {
                        case DOUBLE: {
                            idx.setDoubleNext((Double) o);
                            break;
                        }
                        case FLOAT: {
                            idx.setFloatNext((Float) o);
                            break;
                        }
                        case INTEGER: {
                            idx.setIntNext((Integer) o);
                            break;
                        }
                        case LONG: {
                            idx.setLongNext((Long) o);
                            break;
                        }
                        case BYTE: {
                            idx.setByteNext((Byte) o);
                            break;
                        }
                        case SHORT: {
                            idx.setShortNext((Short) o);
                            break;
                        }
                        case BOOLEAN: {
                            idx.setBooleanNext(Boolean.parseBoolean(st.sval));
                            break;
                        }
                        case STRING: {
                            idx.setObjectNext(st.sval);
                            break;
                        }
                        case OBJECT: {
                            throw new IllegalArgumentException("Could not handle type");
                        }
                        }
                    }
                }
            }
        } catch (final IOException e) {
            log.warn("Could not read data for {}", name);
        }
        if (a != null && ranges != null && ranges.length != 0) {
            try {
                return a.section(Arrays.asList(ranges));
            } catch (InvalidRangeException ex) {
                log.warn("Invalid range while trying to subset array: ", ex);
            }
        }
        return a;
    }

    /**
     *
     * @param root
     * @return
     */
    protected IFileFragment handleFile(final Element root) {
        final Element file = root.getChild("file");
        final Attribute filename1 = file.getAttribute("filename");
        final Attribute dirname = file.getAttribute("dirname");
        // Attribute resourceLocation = file.getAttribute("resourceLocation");
        log.debug("Associated file is {} {}", dirname.getValue(), filename1.getValue());
        final File f = new File(dirname.getValue(), filename1.getValue());
        final IFileFragment ff1 = new FileFragment(f.toURI());
        handleAttributes(ff1, file);
        final List<?> l = file.getChildren("namedGroup");
        final HashSet<String> idxVars = new HashSet<>();
        // first pass: read all non indexed variables first
        for (final Object o : l) {
            final Element group = (Element) o;
            final String varname = group.getAttribute("name").getValue();
            final Element idxVar = group.getChild("indexVariable");
            if (idxVar != null) {
                idxVars.add(varname);
            } else {
                final IVariableFragment ngf = handleVariable(ff1, group);
                EvalTools.notNull(ngf, this);
            }
        }
        // second pass: initialize all indexed variables and set indices
        for (final Object o : l) {
            final Element group = (Element) o;
            final String idxVarName = group.getChild("indexVariable").getAttribute("name").getName();
            final IVariableFragment ngf = handleVariable(ff1, group);
            ngf.setIndex(ff1.getChild(idxVarName));
            EvalTools.notNull(ngf, this);
        }
        return ff1;
    }

    /**
     *
     * @param parent
     * @param var
     * @return
     */
    protected IVariableFragment handleVariable(final IFileFragment parent, final Element var) {
        final String varname = var.getAttribute("name").getValue();
        if (parent.hasChild(varname)) {
            return parent.getChild(varname);
        }
        final IVariableFragment vf = parseVariable(parent, var);
        return vf;
    }

    /**
     *
     * @param parent
     * @param var
     * @return
     */
    protected IVariableFragment parseVariable(final IFileFragment parent, final Element var) {
        final String name = var.getAttribute("name").getValue();
        final String dataType = var.getAttribute("dataType").getValue();
        DataType dt = DataType.getType(dataType);
        if (dt == null) {
            dt = DataType.DOUBLE;
        }

        final Dimension[] dims = parseVariableDimensions(var);

        final Element ranges = var.getChild("ranges");
        final List<?> l = ranges.getChildren("range");
        final Range[] rangeA = new Range[l.size()];
        int i = 0;
        for (final Object o : l) {
            final Element range = (Element) o;
            Range r = null;
            if (range != null) {
                final Attribute r_name = range.getAttribute("name");
                final Attribute r_first = range.getAttribute("first");
                final Attribute r_stride = range.getAttribute("stride");
                final Attribute r_last = range.getAttribute("last");
                try {
                    r = new Range(r_name.getValue(), r_first.getIntValue(), r_last.getIntValue(),
                            r_stride.getIntValue());
                } catch (final DataConversionException | InvalidRangeException e) {
                    log.error(e.getLocalizedMessage());
                }
                // if (r_name != null) {
                // r.setName(r_name.getValue());
                // }

                rangeA[i++] = r;
            }
        }

        final IVariableFragment vf = new VariableFragment(parent, name);
        vf.setDimensions(dims);
        vf.setDataType(dt);
        vf.setRange(rangeA);
        handleAttributes(vf, var);
        handleData(name, dims, var, rangeA);
        // VariableFragment vf =
        // FragmentTools.getVariable(parent,name,null,dims,dt,r);

        return vf;
    }

    /**
     * @param var
     * @return
     */
    private Dimension[] parseVariableDimensions(final Element var) {
        final List<?> dimensions = var.getChildren("dimension");
        Dimension[] dims = null;
        if ((dimensions != null) && (dimensions.size() > 0)) {
            dims = new Dimension[dimensions.size()];
            int i = 0;
            for (final Object o : dimensions) {
                final Element dim = (Element) o;
                final Attribute d_length = dim.getAttribute("length");
                // Attribute d_id = dim.getAttribute("id");
                final Attribute d_name = dim.getAttribute("name");
                final Attribute d_shared = dim.getAttribute("shared");
                final Attribute d_unlimited = dim.getAttribute("unlimited");
                final Attribute d_variableLength = dim.getAttribute("variableLength");
                try {
                    dims[i++] = new Dimension(d_name.getValue(), d_length.getIntValue(), d_shared.getBooleanValue(),
                            d_unlimited.getBooleanValue(), d_variableLength.getBooleanValue());
                } catch (final DataConversionException e) {
                    log.error(e.getLocalizedMessage());
                }
            }
        }
        return dims;
    }

    /*
     * (non-Javadoc)
     *
     * @see
     * cross.io.IDataSource#readAll(cross.datastructures.fragments.IFileFragment
     * )
     */
    @Override
    public ArrayList<Array> readAll(final IFileFragment f) throws IOException, ResourceNotAvailableException {
        final File file = FileTools.getFile(f);
        log.info("Deserializing {}", file.getAbsolutePath());
        final SAXBuilder parser = new SAXBuilder();
        try {
            final Document doc = parser.build(new BufferedInputStream(new FileInputStream(file)));
            final Element root = doc.getRootElement();
            final List<?> l = root.getChildren("namedGroup");
            final ArrayList<Array> al = new ArrayList<>();
            for (final Object o : l) {
                final Element group = (Element) o;
                final String name = group.getAttribute("name").getValue();
                IVariableFragment ivf = handleVariable(f, group);
                //            final Dimension[] d = parseVariableDimensions(group);
                al.add(handleData(name, ivf.getDimensions(), group.getChild("data")));
            }
            return al;
        } catch (final JDOMException e) {
            log.error(e.getLocalizedMessage());
        }
        return new ArrayList<>(0);
    }

    /*
     * (non-Javadoc)
     *
     * @see cross.io.IDataSource#readIndexed(cross.datastructures.fragments.
     * IVariableFragment)
     */
    @Override
    public ArrayList<Array> readIndexed(final IVariableFragment f)
            throws IOException, ResourceNotAvailableException {
        EvalTools.notNull(f.getIndex(), this);
        final IVariableFragment index = f.getIndex();
        log.debug("Reading {} with index {}", f.getName(), index.getName());
        // This will be the range of Arrays in the returned ArrayList
        Range[] index_range = index.getRange();
        // Unset the range, so we can read in the full index_array at first
        index.setRange((Range) null);

        // read in the full index_array
        log.debug("Reading index array {}", index);
        Array index_array = readSingle(index);
        switch (DataType.getType(index_array.getElementType())) {
        case LONG:
            log.warn("Index array contains long values, this is currently only supported up to Integer.MAX_VALUE");
            break;
        case INT:
            break;
        }
        // how many scans are stored in the original array/ how many array
        // pointers are contained in index_array?
        final int num_arrays = index_array.getShape()[0];
        // use the default values of index_range
        int index_start = 0;
        if ((index_range != null) && (index_range[0] != null)) {
            index_start = index_range[0].first();
        }
        int index_end = index_array.getShape()[0] - 1;
        if ((index_range != null) && (index_range[0] != null)) {
            index_end = index_range[0].last();
        }
        int index_stride = 1;
        if ((index_range != null) && (index_range[0] != null)) {
            index_stride = index_range[0].stride();
        }

        log.debug("index_start {}, index_end {}, index_stride {}",
                new Object[] { index_start, index_end, index_stride });

        Array data_array = readSingle(f);
        // get the (first) dimension for that (we expect 1D arrays)
        EvalTools.eqI(1, data_array.getShape().length, this);
        final Dimension data_dim = new Dimension("data_dim", data_array.getShape()[0]);

        // absolute array start and end indices in data_array
        int data_start = index_array.getInt(index_start);
        int data_end = index_array.getInt(index_end);
        int data_stride = 1;

        // set stride, if exists
        if ((f.getRange() != null) && (f.getRange()[0] != null)) {
            data_stride = f.getRange()[0].stride();
        }

        log.debug("data_start {}, data_end {}, data_stride {}", new Object[] { data_start, data_end, data_stride });

        // Create a new index array, which is zero based
        if (index_range == null) {
            index_range = new Range[1];
        }
        if (index_range[0] == null) {
            try {
                index_range[0] = new Range(0, index_end - index_start, index_stride);
                log.debug("index_range[0] = {}", index_range[0]);
            } catch (final InvalidRangeException e) {
                log.error(e.getLocalizedMessage());
            }
        }

        int index_offset = 0;

        // create the ArrayList, which will hold the individual arrays
        final ArrayList<Array> al = new ArrayList<>(num_arrays);

        // Iterate over all arrays in range, starting at relative index 0
        // this can be translated to absolute in index_array via
        // index_start+i, where index_start is the positive offset into
        // index_array
        for (int i = 0; i < index_range[0].length(); i += index_stride) {
            // first element of array index_start+i
            data_start = index_array.getInt((index_start + i));
            // if we have reached the last scan start contained in index_array
            // use the length of the data array -1 as absolute end of last array
            if ((i + index_start + 1) == num_arrays) {
                data_end = data_dim.getLength() - 1;
            } else {
                data_end = index_array.getInt((index_start + i + 1)) - 1;
            }
            //safety net
            data_end = Math.min(data_dim.getLength() - 1, data_end);
            if (data_start > data_end) {
                log.warn(
                        "scan_index contains an invalid last scan offset. Inserting terminating array with length 0!");
                al.add(Array.factory(data_array.getElementType(), new int[0]));
            } else {
                try {
                    log.debug("Reading array {}, from {} to {}", new Object[] { i, data_start, data_end });
                    // Read from data_start to data_end incl. with data_stride
                    final Array ai = data_array
                            .sectionNoReduce(Arrays.asList(new Range(data_start, data_end, data_stride)));
                    // Update new index to the changed value
                    // new_index.set(i, index_offset);
                    // Increase the next start offset by the length of array ai
                    // log.debug("new_index[i] = {}, Index offset =
                    // {}",i,index_offset);
                    index_offset += (data_end - data_start);
                    al.add(ai);
                } catch (final InvalidRangeException e) {
                    log.error(e.getLocalizedMessage());
                }
            }
        }
        EvalTools.notNull(al, this);
        index.setRange(index_range);
        return al;
    }

    /*
     * (non-Javadoc)
     *
     * @see cross.io.IDataSource#readSingle(cross.datastructures.fragments.
     * IVariableFragment)
     */
    @Override
    public Array readSingle(final IVariableFragment f) throws IOException, ResourceNotAvailableException {
        final File file = FileTools.getFile(f.getParent());
        log.info("Deserializing {}", file.getAbsolutePath());
        final SAXBuilder parser = new SAXBuilder();
        try {
            final Document doc = parser.build(new BufferedInputStream(new FileInputStream(file)));
            final Element root = doc.getRootElement();
            final List<?> l = root.getChildren("namedGroup");
            for (final Object o : l) {
                final Element group = (Element) o;
                final String name = group.getAttribute("name").getValue();
                if (name.equals(f.getName())) {
                    return handleData(name, f.getDimensions(), group.getChild("data"));
                }
            }

        } catch (final JDOMException e) {
            log.error(e.getLocalizedMessage());
        }
        throw new ResourceNotAvailableException(
                "Could not locate variable " + f.getName() + " in file " + file.getAbsolutePath());
    }

    /*
     * (non-Javadoc)
     *
     * @see cross.io.IDataSource#readStructure(cross.datastructures.fragments.
     * IFileFragment)
     */
    @Override
    public ArrayList<IVariableFragment> readStructure(final IFileFragment f) throws IOException {
        final File file = FileTools.getFile(f);
        log.info("Deserializing {}", file.getAbsolutePath());
        final SAXBuilder parser = new SAXBuilder();
        try {
            final Document doc = parser.build(new BufferedInputStream(new FileInputStream(file)));
            final Element root = doc.getRootElement();
            final List<?> l = root.getChildren("namedGroup");
            final ArrayList<IVariableFragment> al = new ArrayList<>();
            for (final Object o : l) {
                final Element group = (Element) o;
                al.add(handleVariable(f, group));
            }
            return al;
        } catch (final JDOMException e) {
            log.error(e.getLocalizedMessage());
        }
        throw new ResourceNotAvailableException(
                "Could not locate variable " + f.getName() + " in file " + file.getAbsolutePath());
    }

    /*
     * (non-Javadoc)
     *
     * @see cross.io.IDataSource#readStructure(cross.datastructures.fragments.
     * IVariableFragment)
     */
    @Override
    public IVariableFragment readStructure(final IVariableFragment f)
            throws IOException, ResourceNotAvailableException {
        final File file = FileTools.getFile(f.getParent());
        log.info("Deserializing {}", file.getAbsolutePath());
        final SAXBuilder parser = new SAXBuilder();
        try {
            final Document doc = parser.build(new BufferedInputStream(new FileInputStream(file)));
            final Element root = doc.getRootElement();
            final List<?> l = root.getChildren("namedGroup");
            for (final Object o : l) {
                final Element group = (Element) o;
                if (group.getAttribute("name").getValue().equals(f.getName())) {
                    return handleVariable(f.getParent(), group);
                }
            }
        } catch (final JDOMException e) {
            log.error(e.getLocalizedMessage());
        }
        throw new ResourceNotAvailableException(
                "Could not locate variable " + f.getName() + " in file " + file.getAbsolutePath());
    }

    /**
     *
     * @param iff
     * @return
     * @throws IOException
     */
    public String serialize(final IFileFragment iff) throws IOException {
        final String filename = StringTools.removeFileExt(FileTools.getFilename(iff.getUri())) + ".maltcms.xml";
        final Element maltcms = new Element("maltcms");
        final Document doc = new Document(maltcms);
        iff.appendXML(maltcms);
        final XMLOutputter outp = new XMLOutputter();
        outp.output(doc, new BufferedOutputStream(new FileOutputStream(new File(filename))));
        return filename;
    }

    /*
     * (non-Javadoc)
     *
     * @see cross.io.IDataSource#supportedFormats()
     */
    @Override
    public List<String> supportedFormats() {
        return Arrays.asList(new String[] { "maltcms.xml" });
    }

    /*
     * (non-Javadoc)
     *
     * @see
     * cross.io.IDataSource#write(cross.datastructures.fragments.IFileFragment)
     */
    @Override
    public boolean write(final IFileFragment f) {
        try {
            serialize(f);
            return true;
        } catch (final IOException ioex) {
            log.error("{}", ioex.getLocalizedMessage());
            return false;
        }
    }
    // protected IndexFragment handleIndex(FileFragment parent, VariableFragment
    // vf, Element indexVar) {
    // Attribute i_name = indexVar.getAttribute("name");
    // }
    // protected NamedGroupFragment handleGroup(FileFragment parent, Element
    // group) {
    // Attribute gID = group.getAttribute("groupID");
    // Attribute name = group.getAttribute("name");
    // try {
    // long gid = gID.getLongValue();
    // NamedGroupFragment ngf = new
    // NamedGroupFragment(ff,(name==null?""+gid:name.getValue()));
    // handleAttributes(ngf,group);
    // List vars = group.getChildren("variable");
    // for(Object var:vars) {
    // Element variable = (Element)var;
    // handleVariable(parent,ngf,group,variable);
    // }
    // return ngf;
    // } catch (DataConversionException e1) {
    // log.error(e1.getLocalizedMessage());
    // }
    // return null;
    // }
}