ca.nrc.cadc.sc2pkg.PackageIntTest.java Source code

Java tutorial

Introduction

Here is the source code for ca.nrc.cadc.sc2pkg.PackageIntTest.java

Source

/*
************************************************************************
*******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
**************  CENTRE CANADIEN DE DONNES ASTRONOMIQUES  **************
*
*  (c) 2011.                            (c) 2011.
*  Government of Canada                 Gouvernement du Canada
*  National Research Council            Conseil national de recherches
*  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
*  All rights reserved                  Tous droits rservs
*
*  NRC disclaims any warranties,        Le CNRC dnie toute garantie
*  expressed, implied, or               nonce, implicite ou lgale,
*  statutory, of any kind with          de quelque nature que ce
*  respect to the software,             soit, concernant le logiciel,
*  including without limitation         y compris sans restriction
*  any warranty of merchantability      toute garantie de valeur
*  or fitness for a particular          marchande ou de pertinence
*  purpose. NRC shall not be            pour un usage particulier.
*  liable in any event for any          Le CNRC ne pourra en aucun cas
*  damages, whether direct or           tre tenu responsable de tout
*  indirect, special or general,        dommage, direct ou indirect,
*  consequential or incidental,         particulier ou gnral,
*  arising from the use of the          accessoire ou fortuit, rsultant
*  software.  Neither the name          de l'utilisation du logiciel. Ni
*  of the National Research             le nom du Conseil National de
*  Council of Canada nor the            Recherches du Canada ni les noms
*  names of its contributors may        de ses  participants ne peuvent
*  be used to endorse or promote        tre utiliss pour approuver ou
*  products derived from this           promouvoir les produits drivs
*  software without specific prior      de ce logiciel sans autorisation
*  written permission.                  pralable et particulire
*                                       par crit.
*
*  This file is part of the             Ce fichier fait partie du projet
*  OpenCADC project.                    OpenCADC.
*
*  OpenCADC is free software:           OpenCADC est un logiciel libre ;
*  you can redistribute it and/or       vous pouvez le redistribuer ou le
*  modify it under the terms of         modifier suivant les termes de
*  the GNU Affero General Public        la GNU Affero General Public
*  License as published by the          License? telle que publie
*  Free Software Foundation,            par la Free Software Foundation
*  either version 3 of the              : soit la version 3 de cette
*  License, or (at your option)         licence, soit ( votre gr)
*  any later version.                   toute version ultrieure.
*
*  OpenCADC is distributed in the       OpenCADC est distribu
*  hope that it will be useful,         dans lespoir quil vous
*  but WITHOUT ANY WARRANTY;            sera utile, mais SANS AUCUNE
*  without even the implied             GARANTIE : sans mme la garantie
*  warranty of MERCHANTABILITY          implicite de COMMERCIALISABILIT
*  or FITNESS FOR A PARTICULAR          ni dADQUATION  UN OBJECTIF
*  PURPOSE.  See the GNU Affero         PARTICULIER. Consultez la Licence
*  General Public License for           Gnrale Publique GNU Affero
*  more details.                        pour plus de dtails.
*
*  You should have received             Vous devriez avoir reu une
*  a copy of the GNU Affero             copie de la Licence Gnrale
*  General Public License along         Publique GNU Affero avec
*  with OpenCADC.  If not, see          OpenCADC ; si ce nest
*  <http://www.gnu.org/licenses/>.      pas le cas, consultez :
*                                       <http://www.gnu.org/licenses/>.
*
*  $Revision: 5 $
*
************************************************************************
*/

package ca.nrc.cadc.sc2pkg;

import ca.nrc.cadc.auth.AuthMethod;
import ca.nrc.cadc.auth.SSLUtil;
import ca.nrc.cadc.caom2.PlaneURI;
import ca.nrc.cadc.caom2.pkg.PackageRunner;
import ca.nrc.cadc.net.HttpDownload;
import ca.nrc.cadc.reg.Standards;
import ca.nrc.cadc.reg.client.RegistryClient;
import ca.nrc.cadc.util.FileUtil;
import ca.nrc.cadc.util.HexUtil;
import ca.nrc.cadc.util.Log4jInit;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.URI;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;

/**
 *
 * @author pdowler
 */
public class PackageIntTest {
    private static final Logger log = Logger.getLogger(PackageIntTest.class);

    static final String SINGLE_ARTIFACT = "caom:IRIS/f212h000/IRAS-12um";
    static final String SINGLE_ARTIFACT2 = "caom:IRIS/f212h000/IRAS-25um";
    static final String SINGLE_ARTIFACT3 = "caom:IRIS/f212h000/IRAS-60um";
    static final String SINGLE_ARTIFACT4 = "caom:IRIS/f212h000/IRAS-100um";

    static final String SINGLE_ARTIFACT_PUB = "ivo://cadc.nrc.ca/IRIS?f212h000/IRAS-12um";

    static final String SERVICE_ID = "ivo://cadc.nrc.ca/sc2pkg";

    static {
        Log4jInit.setLevel("ca.nrc.cadc.caom2.pkg", Level.INFO);
        Log4jInit.setLevel("ca.nrc.cadc.sc2pkg", Level.INFO);
    }

    RegistryClient reg;

    public PackageIntTest() {
        this.reg = new RegistryClient();
        try {
            File crt = FileUtil.getFileFromResource("x509_CADCRegtest1.pem", PackageIntTest.class);
            SSLUtil.initSSL(crt);
            log.debug("initSSL: " + crt);
        } catch (Throwable t) {
            throw new RuntimeException("failed to init SSL", t);
        }

    }

    @Test
    public void testSingleArtifactRedirectCAOM() {
        try {
            URL serviceURL = reg.getServiceURL(URI.create(SERVICE_ID), Standards.PKG_10, AuthMethod.ANON);
            URL url = new URL(serviceURL.toExternalForm() + "?ID=" + SINGLE_ARTIFACT);
            log.info("testSingleArtifactRedirect: " + url);

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            HttpDownload get = new HttpDownload(url, bos);
            get.setFollowRedirects(false);
            get.run();
            Assert.assertNull("throwable", get.getThrowable());
            Assert.assertEquals(303, get.getResponseCode());
            URL rurl = get.getRedirectURL();
            log.info("testSingleArtifactRedirect location: " + rurl);
            Assert.assertNotNull(rurl);
            Assert.assertEquals("/data/pub/IRIS/I212B1H0", rurl.getPath());
        } catch (Exception unexpected) {
            log.error("unexpected exception", unexpected);
            Assert.fail("unexpected exception: " + unexpected);
        }
    }

    @Test
    public void testSingleArtifactRedirectIVO() {
        try {
            URL serviceURL = reg.getServiceURL(URI.create(SERVICE_ID), Standards.PKG_10, AuthMethod.ANON);
            URL url = new URL(serviceURL.toExternalForm() + "?ID=" + SINGLE_ARTIFACT_PUB);
            log.info("testSingleArtifactRedirect: " + url);

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            HttpDownload get = new HttpDownload(url, bos);
            get.setFollowRedirects(false);
            get.run();
            Assert.assertNull("throwable", get.getThrowable());
            Assert.assertEquals(303, get.getResponseCode());
            URL rurl = get.getRedirectURL();
            log.info("testSingleArtifactRedirect location: " + rurl);
            Assert.assertNotNull(rurl);
            Assert.assertEquals("/data/pub/IRIS/I212B1H0", rurl.getPath());
        } catch (Exception unexpected) {
            log.error("unexpected exception", unexpected);
            Assert.fail("unexpected exception: " + unexpected);
        }
    }

    //@Test
    public void testSingleArtifactRedirectAuthCAOM() {
        try {
            URL serviceURL = reg.getServiceURL(URI.create(SERVICE_ID), Standards.PKG_10, AuthMethod.PASSWORD);
            URL url = new URL(serviceURL.toExternalForm() + "?ID=" + SINGLE_ARTIFACT);
            log.info("testSingleArtifactRedirectAuth: " + url);

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            HttpDownload get = new HttpDownload(url, bos);
            get.setRequestProperty("Authorization", "***REDACTED***");
            get.setFollowRedirects(false);
            get.run();
            Assert.assertNull("throwable", get.getThrowable());
            Assert.assertEquals(303, get.getResponseCode());
            URL rurl = get.getRedirectURL();
            log.info("testSingleArtifactRedirectAuth location: " + rurl);
            Assert.assertNotNull(rurl);
            Assert.assertEquals("http", rurl.getProtocol());
            Assert.assertEquals("/data/auth/IRIS/I212B1H0", rurl.getPath());
        } catch (Exception unexpected) {
            log.error("unexpected exception", unexpected);
            Assert.fail("unexpected exception: " + unexpected);
        }
    }

    //@Test
    public void testSingleArtifactRedirectAuthIVO() {
        try {
            URL serviceURL = reg.getServiceURL(URI.create(SERVICE_ID), Standards.PKG_10, AuthMethod.PASSWORD);
            URL url = new URL(serviceURL.toExternalForm() + "?ID=" + SINGLE_ARTIFACT_PUB);
            log.info("testSingleArtifactRedirectAuth: " + url);

            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            HttpDownload get = new HttpDownload(url, bos);
            get.setRequestProperty("Authorization", "***REDACTED***");
            get.setFollowRedirects(false);
            get.run();
            Assert.assertNull("throwable", get.getThrowable());
            Assert.assertEquals(303, get.getResponseCode());
            URL rurl = get.getRedirectURL();
            log.info("testSingleArtifactRedirectAuth location: " + rurl);
            Assert.assertNotNull(rurl);
            Assert.assertEquals("http", rurl.getProtocol());
            Assert.assertEquals("/data/auth/IRIS/I212B1H0", rurl.getPath());
        } catch (Exception unexpected) {
            log.error("unexpected exception", unexpected);
            Assert.fail("unexpected exception: " + unexpected);
        }
    }

    @Test
    public void testMutliID() {
        File tar = null;
        try {
            StringBuilder sb = new StringBuilder();
            sb.append("?ID=").append(SINGLE_ARTIFACT);
            sb.append("&ID=").append(SINGLE_ARTIFACT2);
            sb.append("&ID=").append(SINGLE_ARTIFACT3);
            sb.append("&ID=").append(SINGLE_ARTIFACT4);
            URL serviceURL = reg.getServiceURL(URI.create(SERVICE_ID), Standards.PKG_10, AuthMethod.ANON);
            URL url = new URL(serviceURL.toExternalForm() + sb.toString());
            File tmp = new File(System.getProperty("java.io.tmpdir"));

            log.info("testMutliID: " + url + " -> " + tmp.getAbsolutePath());

            HttpDownload get = new HttpDownload(url, tmp);
            get.setOverwrite(true); // file exists from create above
            get.run();
            Assert.assertNull("throwable", get.getThrowable());
            Assert.assertEquals(200, get.getResponseCode());
            tar = get.getFile();
            Assert.assertTrue("tar file exists", tar.exists());

            // validate
            List<String> expectedFiles = new ArrayList<String>();
            expectedFiles.add("IRIS-f212h000-IRAS-12um/I212B1H0.fits");
            expectedFiles.add("IRIS-f212h000-IRAS-25um/I212B2H0.fits");
            expectedFiles.add("IRIS-f212h000-IRAS-60um/I212B3H0.fits");
            expectedFiles.add("IRIS-f212h000-IRAS-100um/I212B4H0.fits");
            expectedFiles.add("IRIS-f212h000-IRAS-12um/README");
            expectedFiles.add("IRIS-f212h000-IRAS-25um/README");
            expectedFiles.add("IRIS-f212h000-IRAS-60um/README");
            expectedFiles.add("IRIS-f212h000-IRAS-100um/README");

            // extract tar file and check all the files vs the md5sum in README?
            FileInputStream fis = new FileInputStream(tar);
            TarArchiveInputStream tis = new TarArchiveInputStream(fis);
            Content c1 = getEntry(tis);
            Content c2 = getEntry(tis);
            Content c3 = getEntry(tis);
            Content c4 = getEntry(tis);
            Content r1 = getEntry(tis);
            Content r2 = getEntry(tis);
            Content r3 = getEntry(tis);
            Content r4 = getEntry(tis);

            ArchiveEntry te = tis.getNextTarEntry();
            Assert.assertNull(te);

            Assert.assertTrue(expectedFiles.contains(c1.name));
            Assert.assertTrue(expectedFiles.contains(c2.name));
            Assert.assertTrue(expectedFiles.contains(c3.name));
            Assert.assertTrue(expectedFiles.contains(c4.name));
            Assert.assertTrue(expectedFiles.contains(r1.name));
            Assert.assertTrue(expectedFiles.contains(r2.name));
            Assert.assertTrue(expectedFiles.contains(r3.name));
            Assert.assertTrue(expectedFiles.contains(r4.name));

            log.debug("testMutliID: " + c1.name + " " + c1.contentMD5);
            log.debug("testMutliID: " + c2.name + " " + c2.contentMD5);
            log.debug("testMutliID: " + c3.name + " " + c3.contentMD5);
            log.debug("testMutliID: " + c4.name + " " + c4.contentMD5);

            // merge md5map(s)
            Map<String, String> md5map = new HashMap<String, String>();
            md5map.putAll(r1.md5map);
            md5map.putAll(r2.md5map);
            md5map.putAll(r3.md5map);
            md5map.putAll(r4.md5map);
            log.debug("testMutliID: " + md5map.size());

            String c1md5 = md5map.get(getFilename(c1.name));
            String c2md5 = md5map.get(getFilename(c2.name));
            String c3md5 = md5map.get(getFilename(c3.name));
            String c4md5 = md5map.get(getFilename(c4.name));

            Assert.assertNotNull(c1md5);
            Assert.assertEquals(c1md5, c1.contentMD5);

            Assert.assertNotNull(c2md5);
            Assert.assertEquals(c2md5, c2.contentMD5);

            Assert.assertNotNull(c3md5);
            Assert.assertEquals(c3md5, c3.contentMD5);

            // cleanup here so we leave file behind on fail
            if (tar.exists())
                tar.delete();
        } catch (Exception unexpected) {
            log.error("unexpected exception", unexpected);
            Assert.fail("unexpected exception: " + unexpected);
        }
    }

    class Content {
        String name;
        String contentMD5;

        Map<String, String> md5map = new HashMap<String, String>();
    }

    private String getFilename(String entryName) {
        int i = entryName.lastIndexOf('/');
        return entryName.substring(i + 1);
    }

    private Content getEntry(TarArchiveInputStream tar) throws IOException, NoSuchAlgorithmException {
        Content ret = new Content();

        TarArchiveEntry entry = tar.getNextTarEntry();
        ret.name = entry.getName();

        if (ret.name.endsWith("README")) {
            byte[] buf = new byte[(int) entry.getSize()];
            tar.read(buf);
            ByteArrayInputStream bis = new ByteArrayInputStream(buf);
            LineNumberReader r = new LineNumberReader(new InputStreamReader(bis));
            String line = r.readLine();
            while (line != null) {
                String[] tokens = line.split(" ");
                // status [md5 filename url]
                String status = tokens[0];
                if ("OK".equals(status)) {
                    String fname = tokens[1];
                    String md5 = tokens[2];
                    ret.md5map.put(fname, md5);
                } else {
                    throw new RuntimeException("tar content failure: " + line);
                }
                line = r.readLine();
            }
        } else {
            MessageDigest md5 = MessageDigest.getInstance("MD5");
            byte[] buf = new byte[8192];
            int n = tar.read(buf);
            while (n > 0) {
                md5.update(buf, 0, n);
                n = tar.read(buf);
            }
            byte[] md5sum = md5.digest();
            ret.contentMD5 = HexUtil.toHex(md5sum);
        }

        return ret;
    }
}