freenet.client.async.ContainerInserter.java Source code

Java tutorial

Introduction

Here is the source code for freenet.client.async.ContainerInserter.java

Source

/* This code is part of Freenet. It is distributed under the GNU General
 * Public License, version 2 (or at your option any later version). See
 * http://www.gnu.org/ for further details of the GPL. */
package freenet.client.async;

import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;

import freenet.client.ClientMetadata;
import freenet.client.DefaultMIMETypes;
import freenet.client.InsertBlock;
import freenet.client.InsertContext;
import freenet.client.InsertException;
import freenet.client.InsertException.InsertExceptionMode;
import freenet.client.Metadata;
import freenet.client.MetadataUnresolvedException;
import freenet.client.ArchiveManager.ARCHIVE_TYPE;
import freenet.client.Metadata.DocumentType;
import freenet.client.Metadata.SimpleManifestComposer;
import freenet.client.async.BaseManifestPutter.PutHandler;
import freenet.keys.FreenetURI;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.api.ManifestElement;
import freenet.support.api.RandomAccessBucket;
import freenet.support.io.BucketTools;
import freenet.support.io.Closer;
import freenet.support.io.ResumeFailedException;

/**
 * Insert a bunch of files as single Archive with .metadata
 * pack the container/archive, then hand it off to SimpleFileInserter
 *
 * TODO persistence
 * TODO add a MAX_SIZE for the final container(file)
 * 
 * @author saces
 * 
 */
public class ContainerInserter implements ClientPutState, Serializable {

    private static final long serialVersionUID = 1L;
    private static volatile boolean logMINOR;
    private static volatile boolean logDEBUG;

    static {
        Logger.registerClass(ContainerInserter.class);
    }

    private static class ContainerElement {
        private final Bucket data;
        private final String targetInArchive;

        private ContainerElement(Bucket data2, String targetInArchive2) {
            data = data2;
            targetInArchive = targetInArchive2;
        }
    }

    private ArrayList<ContainerElement> containerItems;

    private final BaseClientPutter parent;
    private final PutCompletionCallback cb;
    private boolean cancelled;
    private boolean finished;
    private final boolean persistent;
    /** See ContainerBuilder._rootDir. */
    private final HashMap<String, Object> origMetadata;
    private final ARCHIVE_TYPE archiveType;
    private final FreenetURI targetURI;
    private final Object token;
    private final InsertContext ctx;
    private final boolean reportMetadataOnly;
    private final boolean dontCompress;
    final byte[] forceCryptoKey;
    final byte cryptoAlgorithm;
    private final boolean realTimeFlag;

    /**
     * Insert a bunch of files as single Archive with .metadata
     * 
     * @param parent2
     * @param cb2
     * @param metadata2
     * @param targetURI2 The caller need to clone it for persistance
     * @param ctx2
     * @param dontCompress2
     * @param reportMetadataOnly2
     * @param token2
     * @param archiveType2
     * @param freeData
     * @param forceCryptoKey
     * @param cryptoAlgorithm
     * @param realTimeFlag
     * 
     */
    public ContainerInserter(BaseClientPutter parent2, PutCompletionCallback cb2, HashMap<String, Object> metadata2,
            FreenetURI targetURI2, InsertContext ctx2, boolean dontCompress2, boolean reportMetadataOnly2,
            Object token2, ARCHIVE_TYPE archiveType2, boolean freeData, byte[] forceCryptoKey, byte cryptoAlgorithm,
            boolean realTimeFlag) {
        parent = parent2;
        cb = cb2;
        hashCode = super.hashCode();
        persistent = parent.persistent();
        origMetadata = metadata2;
        archiveType = archiveType2;
        targetURI = targetURI2;
        token = token2;
        ctx = ctx2;
        dontCompress = dontCompress2;
        reportMetadataOnly = reportMetadataOnly2;
        containerItems = new ArrayList<ContainerElement>();
        this.forceCryptoKey = forceCryptoKey;
        this.cryptoAlgorithm = cryptoAlgorithm;
        this.realTimeFlag = realTimeFlag;
    }

    @Override
    public void cancel(ClientContext context) {
        synchronized (this) {
            if (cancelled)
                return;
            cancelled = true;
        }
        // Must call onFailure so get removeFrom()'ed
        cb.onFailure(new InsertException(InsertExceptionMode.CANCELLED), this, context);
    }

    @Override
    public BaseClientPutter getParent() {
        return parent;
    }

    @Override
    public Object getToken() {
        return token;
    }

    @Override
    public void schedule(ClientContext context) throws InsertException {
        start(context);
    }

    private void start(ClientContext context) {
        if (logDEBUG)
            Logger.debug(this, "Atempt to start a container inserter", new Exception("debug"));

        makeMetadata(context);

        synchronized (this) {
            if (finished)
                return;
        }

        InsertBlock block;
        OutputStream os = null;
        try {
            RandomAccessBucket outputBucket = context.getBucketFactory(persistent).makeBucket(-1);
            os = new BufferedOutputStream(outputBucket.getOutputStream());
            String mimeType = (archiveType == ARCHIVE_TYPE.TAR ? createTarBucket(os) : createZipBucket(os));
            os = null; // create*Bucket closes os
            if (logMINOR)
                Logger.minor(this, "Archive size is " + outputBucket.size());

            if (logMINOR)
                Logger.minor(this, "We are using " + archiveType);

            // Now we have to insert the Archive we have generated.

            // Can we just insert it, and not bother with a redirect to it?
            // Thereby exploiting implicit manifest support, which will pick up on .metadata??
            // We ought to be able to !!
            block = new InsertBlock(outputBucket, new ClientMetadata(mimeType), targetURI);
        } catch (IOException e) {
            fail(new InsertException(InsertExceptionMode.BUCKET_ERROR, e, null), context);
            return;
        } finally {
            Closer.close(os);
        }

        boolean dc = dontCompress;
        if (!dontCompress) {
            dc = (archiveType == ARCHIVE_TYPE.ZIP);
        }

        // Treat it as a splitfile for purposes of determining reinsert count.
        SingleFileInserter sfi = new SingleFileInserter(parent, cb, block, false, ctx, realTimeFlag, dc,
                reportMetadataOnly, token, archiveType, true, null, true, persistent, 0, 0, null, cryptoAlgorithm,
                forceCryptoKey, -1);
        if (logMINOR)
            Logger.minor(this, "Inserting container: " + sfi + " for " + this);
        cb.onTransition(this, sfi, context);
        try {
            sfi.schedule(context);
        } catch (InsertException e) {
            fail(new InsertException(InsertExceptionMode.BUCKET_ERROR, e, null), context);
            return;
        }
    }

    private void makeMetadata(ClientContext context) {

        Bucket bucket = null;
        int x = 0;

        Metadata md = makeManifest(origMetadata, "");

        while (true) {
            try {
                bucket = md.toBucket(context.getBucketFactory(persistent));
                containerItems.add(new ContainerElement(bucket, ".metadata"));
                return;
            } catch (MetadataUnresolvedException e) {
                try {
                    x = resolve(e, x, null, null, context);
                } catch (IOException e1) {
                    fail(new InsertException(InsertExceptionMode.INTERNAL_ERROR, e, null), context);
                    return;
                }
            } catch (IOException e) {
                fail(new InsertException(InsertExceptionMode.INTERNAL_ERROR, e, null), context);
                return;
            }
        }

    }

    private int resolve(MetadataUnresolvedException e, int x, FreenetURI key, String element2,
            ClientContext context) throws IOException {
        Metadata[] metas = e.mustResolve;
        for (Metadata m : metas) {
            try {
                Bucket bucket = m.toBucket(context.getBucketFactory(persistent));
                String nameInArchive = ".metadata-" + (x++);
                containerItems.add(new ContainerElement(bucket, nameInArchive));
                m.resolve(nameInArchive);
            } catch (MetadataUnresolvedException e1) {
                x = resolve(e, x, key, element2, context);
            }
        }
        return x;
    }

    private void fail(InsertException e, ClientContext context) {
        // Cancel all, then call the callback
        synchronized (this) {
            if (finished)
                return;
            finished = true;
        }
        cb.onFailure(e, this, context);
    }

    // A persistent hashCode is helpful in debugging, and also means we can put
    // these objects into sets etc when we need to.

    private final int hashCode;

    @Override
    public int hashCode() {
        return hashCode;
    }

    /**
    ** OutputStream os will be close()d if this method returns successfully.
    */
    private String createTarBucket(OutputStream os) throws IOException {
        if (logMINOR)
            Logger.minor(this, "Create a TAR Bucket");

        TarArchiveOutputStream tarOS = new TarArchiveOutputStream(os);
        try {
            tarOS.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
            TarArchiveEntry ze;

            for (ContainerElement ph : containerItems) {
                if (logMINOR)
                    Logger.minor(this, "Putting into tar: " + ph + " data length " + ph.data.size() + " name "
                            + ph.targetInArchive);
                ze = new TarArchiveEntry(ph.targetInArchive);
                ze.setModTime(0);
                long size = ph.data.size();
                ze.setSize(size);
                tarOS.putArchiveEntry(ze);
                BucketTools.copyTo(ph.data, tarOS, size);
                tarOS.closeArchiveEntry();
            }
        } finally {
            tarOS.close();
        }

        return ARCHIVE_TYPE.TAR.mimeTypes[0];
    }

    private String createZipBucket(OutputStream os) throws IOException {
        if (logMINOR)
            Logger.minor(this, "Create a ZIP Bucket");

        ZipOutputStream zos = new ZipOutputStream(os);
        try {
            ZipEntry ze;

            for (ContainerElement ph : containerItems) {
                ze = new ZipEntry(ph.targetInArchive);
                ze.setTime(0);
                zos.putNextEntry(ze);
                BucketTools.copyTo(ph.data, zos, ph.data.size());
                zos.closeEntry();
            }
        } finally {
            zos.close();
        }

        return ARCHIVE_TYPE.ZIP.mimeTypes[0];
    }

    private Metadata makeManifest(HashMap<String, Object> manifestElements, String archivePrefix) {
        SimpleManifestComposer smc = new Metadata.SimpleManifestComposer();
        for (Map.Entry<String, Object> me : manifestElements.entrySet()) {
            String name = me.getKey();
            Object o = me.getValue();
            if (o instanceof HashMap) {
                @SuppressWarnings("unchecked")
                HashMap<String, Object> hm = (HashMap<String, Object>) o;
                HashMap<String, Object> subMap = new HashMap<String, Object>();
                //System.out.println("Decompose: "+name+" (SubDir)");
                smc.addItem(name, makeManifest(hm, archivePrefix + name + '/'));
                if (logDEBUG)
                    Logger.debug(this,
                            "Sub map for " + name + " : " + subMap.size() + " elements from " + hm.size());
            } else if (o instanceof Metadata) {
                //already Metadata, take it as is
                //System.out.println("Decompose: "+name+" (Metadata)");
                smc.addItem(name, (Metadata) o);
            } else {
                ManifestElement element = (ManifestElement) o;
                String mimeType = element.getMimeType();
                ClientMetadata cm;
                if (mimeType == null || mimeType.equals(DefaultMIMETypes.DEFAULT_MIME_TYPE))
                    cm = null;
                else
                    cm = new ClientMetadata(mimeType);
                Metadata m;
                if (element.targetURI != null) {
                    //System.out.println("Decompose: "+name+" (ManifestElement, Redirect)");
                    m = new Metadata(DocumentType.SIMPLE_REDIRECT, null, null, element.targetURI, cm);
                } else {
                    //System.out.println("Decompose: "+name+" (ManifestElement, Data)");
                    containerItems.add(new ContainerElement(element.getData(), archivePrefix + name));
                    m = new Metadata(DocumentType.ARCHIVE_INTERNAL_REDIRECT, null, null,
                            archivePrefix + element.fullName, cm);
                }
                smc.addItem(name, m);
            }
        }
        return smc.getMetadata();
    }

    private transient boolean resumed = false;

    @Override
    public void onResume(ClientContext context) throws InsertException, ResumeFailedException {
        synchronized (this) {
            if (resumed)
                return;
            resumed = true;
        }
        if (cb != null && cb != parent)
            cb.onResume(context);
        if (containerItems != null) {
            for (ContainerElement e : containerItems) {
                if (e.data != null)
                    e.data.onResume(context);
            }
        }
        resumeMetadata(origMetadata, context);
        // Do not call start(). start() immediately transitions to another state.
    }

    @SuppressWarnings("unchecked")
    public static void resumeMetadata(Map<String, Object> map, ClientContext context) throws ResumeFailedException {
        Map<String, Object> manifestElements = (Map<String, Object>) map;
        for (Object o : manifestElements.values()) {
            if (o instanceof HashMap) {
                resumeMetadata((Map<String, Object>) o, context);
            } else if (o instanceof ManifestElement) {
                ManifestElement e = (ManifestElement) o;
                e.onResume(context);
            } else if (o instanceof Metadata) {
                // Ignore
            } else if (o instanceof PutHandler) {
                PutHandler handler = (PutHandler) o;
                handler.onResume(context);
            } else if (o instanceof ManifestElement) {
                ((ManifestElement) o).onResume(context);
            } else
                throw new IllegalArgumentException("Unknown manifest element: " + o);
        }
    }

    @Override
    public void onShutdown(ClientContext context) {
        // Ignore.
    }
}