Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.usergrid.management.export; import java.io.File; import java.io.FileReader; import java.util.*; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.util.concurrent.Service; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.usergrid.batch.service.JobSchedulerService; import org.apache.usergrid.utils.UUIDUtils; import org.jclouds.ContextBuilder; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule; import org.jclouds.logging.log4j.config.Log4JLoggingModule; import org.jclouds.netty.config.NettyPayloadModule; import org.junit.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.usergrid.NewOrgAppAdminRule; import org.apache.usergrid.ServiceITSetup; import org.apache.usergrid.ServiceITSetupImpl; import org.apache.usergrid.batch.JobExecution; import org.apache.usergrid.cassandra.ClearShiroSubject; import org.apache.usergrid.management.ApplicationInfo; import org.apache.usergrid.management.OrganizationInfo; import org.apache.usergrid.management.UserInfo; import org.apache.usergrid.persistence.Entity; import org.apache.usergrid.persistence.EntityManager; import org.apache.usergrid.persistence.SimpleEntityRef; import org.apache.usergrid.persistence.entities.JobData; import org.apache.usergrid.setup.ConcurrentProcessSingleton; import com.amazonaws.SDKGlobalConfiguration; import com.google.common.collect.ImmutableSet; import com.google.inject.Module; import static org.apache.usergrid.TestHelper.newUUIDString; import static org.apache.usergrid.TestHelper.uniqueApp; import static org.apache.usergrid.TestHelper.uniqueOrg; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** * * */ public class ExportServiceIT { private static final Logger logger = LoggerFactory.getLogger(ExportServiceIT.class); @ClassRule public static final ServiceITSetup setup = new ServiceITSetupImpl(); @Rule public ClearShiroSubject clearShiroSubject = new ClearShiroSubject(); @Rule public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule(setup); // app-level data generated only once private UserInfo adminUser; private OrganizationInfo organization; private UUID applicationId; private static String bucketPrefix; private String bucketName; @Before public void setup() throws Exception { logger.info("in setup"); // start the scheduler after we're all set up try { JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource() .getBean(JobSchedulerService.class); if (jobScheduler.state() != Service.State.RUNNING) { jobScheduler.startAsync(); jobScheduler.awaitRunning(); } } catch (Exception e) { logger.warn("Ignoring error starting jobScheduler, already started?", e); } adminUser = newOrgAppAdminRule.getAdminInfo(); organization = newOrgAppAdminRule.getOrganizationInfo(); applicationId = newOrgAppAdminRule.getApplicationInfo().getId(); setup.getEntityIndex().refresh(applicationId); } @Before public void before() { boolean configured = !StringUtils.isEmpty(System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR)) && !StringUtils.isEmpty(System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR)) && !StringUtils.isEmpty(System.getProperty("bucketName")); if (!configured) { logger.warn( "Skipping test because {}, {} and bucketName not " + "specified as system properties, e.g. in your Maven settings.xml file.", new Object[] { SDKGlobalConfiguration.SECRET_KEY_ENV_VAR, SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR }); } Assume.assumeTrue(configured); adminUser = newOrgAppAdminRule.getAdminInfo(); organization = newOrgAppAdminRule.getOrganizationInfo(); applicationId = newOrgAppAdminRule.getApplicationInfo().getId(); bucketPrefix = System.getProperty("bucketName"); bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase(); } //Tests to make sure we can call the job with mock data and it runs. // Connections won't save when run with maven, but on local builds it will. @Ignore("Pending merge of export-feature branch") public void testConnectionsOnCollectionExport() throws Exception { File f = null; int indexCon = 0; try { f = new File("testFileConnections.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't then don't do anything and carry on. } f.deleteOnExit(); S3Export s3Export = new MockS3ExportImpl("testFileConnections.json"); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); payload.put("collectionName", "users"); EntityManager em = setup.getEmf().getEntityManager(applicationId); //intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[2]; //creates entities for (int i = 0; i < 2; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "meatIsGreat" + i); userProperties.put("email", "grey" + i + "@anuff.com");//String.format( "test%i@anuff.com", i ) ); entity[i] = em.create("users", userProperties); } //creates connections em.createConnection(em.get(new SimpleEntityRef("user", entity[0].getUuid())), "Vibrations", em.get(new SimpleEntityRef("user", entity[1].getUuid()))); em.createConnection(em.get(new SimpleEntityRef("user", entity[1].getUuid())), "Vibrations", em.get(new SimpleEntityRef("user", entity[0].getUuid()))); UUID exportUUID = exportService.schedule(payload); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); HashMap<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); List usersList = (List) collectionsMap.get("users"); int indexApp = 0; for (indexApp = 0; indexApp < usersList.size(); indexApp++) { Map user = (Map) usersList.get(indexApp); Map userProps = (Map) user.get("Metadata"); String uuid = (String) userProps.get("uuid"); if (entity[0].getUuid().toString().equals(uuid)) { break; } } assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size()); Map userMap = (Map) usersList.get(indexApp); Map connectionsMap = (Map) userMap.get("connections"); assertNotNull(connectionsMap); List vibrationsList = (List) connectionsMap.get("Vibrations"); assertNotNull(vibrationsList); f.deleteOnExit(); } @Test //Connections won't save when run with maven, but on local builds it will. public void testConnectionsOnApplicationEndpoint() throws Exception { File f = null; try { f = new File("testConnectionsOnApplicationEndpoint.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't then don't do anything and carry on. } String fileName = "testConnectionsOnApplicationEndpoint.json"; S3Export s3Export = new MockS3ExportImpl("testConnectionsOnApplicationEndpoint.json"); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); EntityManager em = setup.getEmf().getEntityManager(applicationId); // intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[2]; // creates entities for (int i = 0; i < 2; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "billybob" + i); userProperties.put("email", "test" + i + "@anuff.com");//String.format( "test%i@anuff.com", i ) ); entity[i] = em.create("users", userProperties); } setup.getEntityIndex().refresh(applicationId); //creates connections em.createConnection(em.get(new SimpleEntityRef("user", entity[0].getUuid())), "Vibrations", em.get(new SimpleEntityRef("user", entity[1].getUuid()))); em.createConnection(em.get(new SimpleEntityRef("user", entity[1].getUuid())), "Vibrations", em.get(new SimpleEntityRef("user", entity[0].getUuid()))); UUID exportUUID = exportService.schedule(payload); //create and initialize jobData returned in JobExecution. JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); HashMap<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); List usersList = (List) collectionsMap.get("users"); int indexApp = 0; for (indexApp = 0; indexApp < usersList.size(); indexApp++) { Map user = (Map) usersList.get(indexApp); Map userProps = (Map) user.get("Metadata"); String uuid = (String) userProps.get("uuid"); if (entity[0].getUuid().toString().equals(uuid)) { break; } } assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size()); Map userMap = (Map) usersList.get(indexApp); Map connectionsMap = (Map) userMap.get("connections"); assertNotNull(connectionsMap); List vibrationsList = (List) connectionsMap.get("Vibrations"); assertNotNull(vibrationsList); f.deleteOnExit(); } @Test public void testExportOneOrgCollectionEndpoint() throws Exception { File f = null; try { f = new File("exportOneOrg.json"); } catch (Exception e) { //consumed because this checks to see if the file exists. // If it doesn't then don't do anything and carry on. } //create another org to ensure we don't export it newOrgAppAdminRule.createOwnerAndOrganization("noExport" + newUUIDString(), "junkUserName" + newUUIDString(), "junkRealName" + newUUIDString(), newUUIDString() + "ugExport@usergrid.com", "123456789"); S3Export s3Export = new MockS3ExportImpl("exportOneOrg.json"); // s3Export.setFilename( "exportOneOrg.json" ); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); payload.put("collectionName", "roles"); UUID exportUUID = exportService.schedule(payload); JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); Map<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); String collectionName = (String) collectionsMap.keySet().iterator().next(); List collection = (List) collectionsMap.get(collectionName); for (Object o : collection) { Map entityMap = (Map) o; Map metadataMap = (Map) entityMap.get("Metadata"); String entityName = (String) metadataMap.get("name"); assertFalse("junkRealName".equals(entityName)); } f.deleteOnExit(); } // //creation of files doesn't always delete itself @Test public void testExportOneAppOnCollectionEndpoint() throws Exception { final String orgName = uniqueOrg(); final String appName = uniqueApp(); File f = null; try { f = new File("exportOneApp.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't, don't do anything and carry on. } f.deleteOnExit(); Entity appInfo = setup.getEmf().createApplicationV2(orgName, appName); UUID applicationId = appInfo.getUuid(); EntityManager em = setup.getEmf().getEntityManager(applicationId); //intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[1]; //creates entities for (int i = 0; i < 1; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "junkRealName"); userProperties.put("email", "test" + i + "@anuff.com"); entity[i] = em.create("user", userProperties); } S3Export s3Export = new MockS3ExportImpl("exportOneApp.json"); //s3Export.setFilename( "exportOneApp.json" ); ExportService exportService = setup.getExportService(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); UUID exportUUID = exportService.schedule(payload); JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); Map<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); String collectionName = (String) collectionsMap.keySet().iterator().next(); List collection = (List) collectionsMap.get(collectionName); for (Object o : collection) { Map entityMap = (Map) o; Map metadataMap = (Map) entityMap.get("Metadata"); String entityName = (String) metadataMap.get("name"); assertFalse("junkRealName".equals(entityName)); } } @Test public void testExportOneAppOnApplicationEndpointWQuery() throws Exception { File f = null; try { f = new File("exportOneAppWQuery.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't, don't do anything and carry on. } f.deleteOnExit(); EntityManager em = setup.getEmf().getEntityManager(applicationId); //intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[1]; //creates entities for (int i = 0; i < 1; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("name", "me"); userProperties.put("username", "junkRealName"); userProperties.put("email", "burp" + i + "@anuff.com"); entity[i] = em.create("users", userProperties); } S3Export s3Export = new MockS3ExportImpl("exportOneAppWQuery.json"); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("query", "select * where username = 'junkRealName'"); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); UUID exportUUID = exportService.schedule(payload); JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); setup.getEntityIndex().refresh(applicationId); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); Map<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); String collectionName = (String) collectionsMap.keySet().iterator().next(); List collection = (List) collectionsMap.get(collectionName); for (Object o : collection) { Map entityMap = (Map) o; Map metadataMap = (Map) entityMap.get("Metadata"); String entityName = (String) metadataMap.get("name"); assertFalse("junkRealName".equals(entityName)); } } @Test public void testExportOneCollection() throws Exception { File f = null; int entitiesToCreate = 5; try { f = new File("exportOneCollection.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't, don't do anything and carry on. } f.deleteOnExit(); EntityManager em = setup.getEmf().getEntityManager(applicationId); // em.createApplicationCollection( "qtsMagics" ); // intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[entitiesToCreate]; //creates entities for (int i = 0; i < entitiesToCreate; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "billybob" + i); userProperties.put("email", "test" + i + "@anuff.com"); entity[i] = em.create("qtsMagics", userProperties); } S3Export s3Export = new MockS3ExportImpl("exportOneCollection.json"); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); payload.put("collectionName", "qtsMagics"); UUID exportUUID = exportService.schedule(payload); JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); setup.getEntityIndex().refresh(applicationId); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); HashMap<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); String collectionName = (String) collectionsMap.keySet().iterator().next(); List collection = (List) collectionsMap.get(collectionName); assertEquals(entitiesToCreate, collection.size()); } @Test public void testExportOneCollectionWQuery() throws Exception { File f = null; int entitiesToCreate = 5; try { f = new File("exportOneCollectionWQuery.json"); } catch (Exception e) { // consumed because this checks to see if the file exists. // If it doesn't, don't do anything and carry on. } f.deleteOnExit(); EntityManager em = setup.getEmf().getEntityManager(applicationId); em.createApplicationCollection("baconators"); setup.getEntityIndex().refresh(applicationId); //initialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[entitiesToCreate]; // creates entities for (int i = 0; i < entitiesToCreate; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "billybob" + i); userProperties.put("email", "test" + i + "@anuff.com"); entity[i] = em.create("baconators", userProperties); } S3Export s3Export = new MockS3ExportImpl("exportOneCollectionWQuery.json"); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("query", "select * where username contains 'billybob0'"); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); payload.put("collectionName", "baconators"); UUID exportUUID = exportService.schedule(payload); JobData jobData = jobDataCreator(payload, exportUUID, s3Export); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); setup.getEntityIndex().refresh(applicationId); exportService.doExport(jobExecution); TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() { }; ObjectMapper mapper = new ObjectMapper(); Map<String, Object> jsonMap = mapper.readValue(new FileReader(f), typeRef); Map collectionsMap = (Map) jsonMap.get("collections"); String collectionName = (String) collectionsMap.keySet().iterator().next(); List collectionList = (List) collectionsMap.get(collectionName); assertEquals(1, collectionList.size()); } @Test public void testExportDoJob() throws Exception { String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); JobData jobData = new JobData(); jobData.setProperty("jobName", "exportJob"); // this needs to be populated with properties of export info jobData.setProperty("exportInfo", payload); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); when(jobExecution.getJobId()).thenReturn(UUID.randomUUID()); ExportJob job = new ExportJob(); ExportService eS = mock(ExportService.class); job.setExportService(eS); try { job.doJob(jobExecution); } catch (Exception e) { logger.error("Error doing job", e); assert (false); } assert (true); } //tests that with empty job data, the export still runs. @Test public void testExportEmptyJobData() throws Exception { JobData jobData = new JobData(); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); when(jobExecution.getJobId()).thenReturn(UUID.randomUUID()); ExportJob job = new ExportJob(); S3Export s3Export = mock(S3Export.class); //setup.getExportService().setS3Export( s3Export ); job.setExportService(setup.getExportService()); try { job.doJob(jobExecution); } catch (Exception e) { assert (false); } assert (true); } @Test public void testNullJobExecution() { JobData jobData = new JobData(); JobExecution jobExecution = mock(JobExecution.class); when(jobExecution.getJobData()).thenReturn(jobData); when(jobExecution.getJobId()).thenReturn(UUID.randomUUID()); ExportJob job = new ExportJob(); S3Export s3Export = mock(S3Export.class); // setup.getExportService().setS3Export( s3Export ); job.setExportService(setup.getExportService()); try { job.doJob(jobExecution); } catch (Exception e) { assert (false); } assert (true); } @Test @Ignore("Pending merge of export-feature branch") public void testIntegration100EntitiesOn() throws Exception { if (logger.isDebugEnabled()) { logger.debug("testIntegration100EntitiesOn(): starting..."); } ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); // create five applications each with collection of five entities for (int i = 0; i < 5; i++) { ApplicationInfo appMade = setup.getMgmtSvc().createApplication(organization.getUuid(), "superapp" + i); EntityManager appEm = setup.getEmf().getEntityManager(appMade.getId()); String collName = "superappCol" + i; appEm.createApplicationCollection(collName); Map<String, Object> entityLevelProperties = null; Entity[] entNotCopied; entNotCopied = new Entity[5]; for (int index = 0; index < 5; index++) { entityLevelProperties = new LinkedHashMap<String, Object>(); entityLevelProperties.put("username", "bobso" + index); entityLevelProperties.put("email", "derp" + index + "@anuff.com"); entNotCopied[index] = appEm.create(collName, entityLevelProperties); } } // export the organization containing those apps and collections UUID exportUUID = exportService.schedule(payload); int maxRetries = 100; int retries = 0; while (!exportService.getState(exportUUID).equals("FINISHED") && retries++ < maxRetries) { Thread.sleep(100); } String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR); String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR); Properties overrides = new Properties(); overrides.setProperty("s3" + ".identity", accessId); overrides.setProperty("s3" + ".credential", secretKey); // test that we can find the file that were exported to S3 BlobStore blobStore = null; try { final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule()); BlobStoreContext context = ContextBuilder.newBuilder("s3").credentials(accessId, secretKey) .modules(MODULES).overrides(overrides).buildView(BlobStoreContext.class); String expectedFileName = ((ExportServiceImpl) exportService) .prepareOutputFileName(organization.getName(), "applications"); blobStore = context.getBlobStore(); if (!blobStore.blobExists(bucketName, expectedFileName)) { blobStore.deleteContainer(bucketName); Assert.fail("Blob does not exist: " + expectedFileName); } Blob bo = blobStore.getBlob(bucketName, expectedFileName); Long numOfFiles = blobStore.countBlobs(bucketName); Long numWeWant = 1L; blobStore.deleteContainer(bucketName); assertEquals(numOfFiles, numWeWant); assertNotNull(bo); } finally { blobStore.deleteContainer(bucketName); } } @Test @Ignore("Pending merge of export-feature branch") public void testIntegration100EntitiesForAllApps() throws Exception { S3Export s3Export = new S3ExportImpl(); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); OrganizationInfo orgMade = null; ApplicationInfo appMade = null; for (int i = 0; i < 5; i++) { orgMade = setup.getMgmtSvc().createOrganization("minorboss" + i, adminUser, true); for (int j = 0; j < 5; j++) { appMade = setup.getMgmtSvc().createApplication(orgMade.getUuid(), "superapp" + j); EntityManager customMaker = setup.getEmf().getEntityManager(appMade.getId()); customMaker.createApplicationCollection("superappCol" + j); //intialize user object to be posted Map<String, Object> entityLevelProperties = null; Entity[] entNotCopied; entNotCopied = new Entity[1]; //creates entities for (int index = 0; index < 1; index++) { entityLevelProperties = new LinkedHashMap<String, Object>(); entityLevelProperties.put("derp", "bacon"); entNotCopied[index] = customMaker.create("superappCol" + j, entityLevelProperties); } } } payload.put("organizationId", orgMade.getUuid()); UUID exportUUID = exportService.schedule(payload); assertNotNull(exportUUID); Thread.sleep(3000); String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR); String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR); Properties overrides = new Properties(); overrides.setProperty("s3" + ".identity", accessId); overrides.setProperty("s3" + ".credential", secretKey); BlobStore blobStore = null; try { final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule()); BlobStoreContext context = ContextBuilder.newBuilder("s3").credentials(accessId, secretKey) .modules(MODULES).overrides(overrides).buildView(BlobStoreContext.class); blobStore = context.getBlobStore(); //Grab Number of files Long numOfFiles = blobStore.countBlobs(bucketName); String expectedFileName = ((ExportServiceImpl) exportService) .prepareOutputFileName(organization.getName(), "applications"); //delete container containing said files Blob bo = blobStore.getBlob(bucketName, expectedFileName); Long numWeWant = 5L; blobStore.deleteContainer(bucketName); //asserts that the correct number of files was transferred over assertEquals(numWeWant, numOfFiles); } finally { blobStore.deleteContainer(bucketName); } } @Test @Ignore("Pending merge of export-feature branch") public void testIntegration100EntitiesOnOneOrg() throws Exception { S3Export s3Export = new S3ExportImpl(); ExportService exportService = setup.getExportService(); String appName = newOrgAppAdminRule.getApplicationInfo().getName(); HashMap<String, Object> payload = payloadBuilder(appName); payload.put("organizationId", organization.getUuid()); payload.put("applicationId", applicationId); OrganizationInfo orgMade = null; ApplicationInfo appMade = null; for (int i = 0; i < 100; i++) { orgMade = setup.getMgmtSvc().createOrganization("largerboss" + i, adminUser, true); appMade = setup.getMgmtSvc().createApplication(orgMade.getUuid(), "superapp" + i); EntityManager customMaker = setup.getEmf().getEntityManager(appMade.getId()); customMaker.createApplicationCollection("superappCol" + i); //intialize user object to be posted Map<String, Object> entityLevelProperties = null; Entity[] entNotCopied; entNotCopied = new Entity[20]; //creates entities for (int index = 0; index < 20; index++) { entityLevelProperties = new LinkedHashMap<String, Object>(); entityLevelProperties.put("username", "bobso" + index); entityLevelProperties.put("email", "derp" + index + "@anuff.com"); entNotCopied[index] = customMaker.create("superappCol", entityLevelProperties); } } EntityManager em = setup.getEmf().getEntityManager(applicationId); //intialize user object to be posted Map<String, Object> userProperties = null; Entity[] entity; entity = new Entity[100]; //creates entities for (int i = 0; i < 100; i++) { userProperties = new LinkedHashMap<String, Object>(); userProperties.put("username", "bido" + i); userProperties.put("email", "bido" + i + "@anuff.com"); entity[i] = em.create("user", userProperties); } UUID exportUUID = exportService.schedule(payload); while (!exportService.getState(exportUUID).equals("FINISHED")) { } String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR); String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR); Properties overrides = new Properties(); overrides.setProperty("s3" + ".identity", accessId); overrides.setProperty("s3" + ".credential", secretKey); Blob bo = null; BlobStore blobStore = null; try { final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule()); BlobStoreContext context = ContextBuilder.newBuilder("s3").credentials(accessId, secretKey) .modules(MODULES).overrides(overrides).buildView(BlobStoreContext.class); String expectedFileName = ((ExportServiceImpl) exportService) .prepareOutputFileName(organization.getName(), "applications"); blobStore = context.getBlobStore(); if (!blobStore.blobExists(bucketName, expectedFileName)) { assert (false); } Long numOfFiles = blobStore.countBlobs(bucketName); Long numWeWant = Long.valueOf(1); assertEquals(numOfFiles, numWeWant); bo = blobStore.getBlob(bucketName, expectedFileName); } catch (Exception e) { assert (false); } assertNotNull(bo); blobStore.deleteContainer(bucketName); } public JobData jobDataCreator(HashMap<String, Object> payload, UUID exportUUID, S3Export s3Export) { JobData jobData = new JobData(); jobData.setProperty("jobName", "exportJob"); jobData.setProperty("exportInfo", payload); jobData.setProperty("exportId", exportUUID); jobData.setProperty("s3Export", s3Export); return jobData; } /*Creates fake payload for testing purposes.*/ public HashMap<String, Object> payloadBuilder(String orgOrAppName) { HashMap<String, Object> payload = new HashMap<String, Object>(); Map<String, Object> properties = new HashMap<String, Object>(); Map<String, Object> storage_info = new HashMap<String, Object>(); storage_info.put(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR, System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR)); storage_info.put(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR, System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR)); storage_info.put("bucket_location", bucketName); properties.put("storage_provider", "s3"); properties.put("storage_info", storage_info); payload.put("path", orgOrAppName); payload.put("properties", properties); return payload; } }