Java tutorial
/* * Copyright 2015 Trento Rise (trentorise.eu) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package eu.trentorise.opendata.jackan.test.ckan; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.collect.LinkedListMultimap; import com.google.common.collect.Multimap; import eu.trentorise.opendata.jackan.SearchResults; import eu.trentorise.opendata.jackan.CkanClient; import eu.trentorise.opendata.jackan.model.CkanDataset; import eu.trentorise.opendata.jackan.exceptions.CkanException; import eu.trentorise.opendata.jackan.exceptions.CkanNotFoundException; import eu.trentorise.opendata.jackan.model.CkanGroup; import eu.trentorise.opendata.jackan.model.CkanLicense; import eu.trentorise.opendata.jackan.model.CkanOrganization; import eu.trentorise.opendata.jackan.CkanQuery; import eu.trentorise.opendata.jackan.model.CkanError; import eu.trentorise.opendata.jackan.model.CkanResource; import eu.trentorise.opendata.jackan.model.CkanTag; import eu.trentorise.opendata.jackan.model.CkanUser; import eu.trentorise.opendata.jackan.test.JackanTestConfig; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.UUID; import java.util.logging.Logger; import static junitparams.JUnitParamsRunner.$; import junitparams.JUnitParamsRunner; import junitparams.Parameters; import org.apache.http.HttpHost; import org.junit.After; import org.junit.Assert; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; /** * Performs integration tests. Many tests here are also used by * {@link CkanTestReporter} * * @author David Leoni */ @RunWith(JUnitParamsRunner.class) public class ReadCkanIT { public static final Logger logger = Logger.getLogger(ReadCkanIT.class.getName()); public static String DATI_TRENTINO = "http://dati.trentino.it"; public static String DATI_TOSCANA = "http://dati.toscana.it"; public static String DATI_MATERA = "http://dati.comune.matera.it"; public static String DATA_GOV_UK = "http://data.gov.uk"; public static String DATA_GOV_US = "http://catalog.data.gov"; /** for testing timeouts */ public static String UNREACHABLE_HOST = "http://10.0.0.0"; /** * National Oceanic and Atmospheric Administration (United States) */ public static String NOAA_GOV_US = "https://data.noaa.gov"; /** * Unfortunately this one uses old api version, we can't use it. */ public static String DATI_PIEMONTE = "http://www.dati.piemonte.it/rpapisrv/api/rest"; public static final String LAGHI_MONITORATI_TRENTO_NAME = "laghi-monitorati-trento-143675"; public static final String LAGHI_MONITORATI_TRENTO_ID = "3745b44c-751f-40b3-8e97-ccd725bfbe8a"; public static final String LAGHI_MONITORATI_TRENTO_XML_RESOURCE_NAME = "Metadati in formato XML"; public static final String PRODOTTI_CERTIFICATI_DATASET_NAME = "prodotti-certificati"; public static final String PRODOTTI_CERTIFICATI_RESOURCE_ID = "fe507a10-4c49-4b18-8bf6-6705198cfd42"; public static final String POLITICHE_SVILUPPO_ORGANIZATION_NAME = "pat-s-sviluppo-rurale"; public static final String AGRICOLTURA_GROUP_NAME = "agricoltura"; private Multimap<String, String> datasetList = LinkedListMultimap.create(); public static int TEST_ELEMENTS = 5; /** * Object mapper for reading */ private ObjectMapper objectMapper; /** * All reading tests will be tried with all these catalogs. */ public Object[] clients() { return $($(new CkanClient(DATI_TRENTINO)), $(new CkanClient(DATI_TOSCANA)) //$(new CkanClient(NOAA_GOV_US)) /*, $(new CkanClient(DATI_MATERA)), $(new CkanClient(DATA_GOV_UK)), $(new CkanClient(DATA_GOV_US)) */ ); } @BeforeClass public static void setUpClass() { JackanTestConfig.of().loadConfig(); } @Before public void setUp() { objectMapper = new ObjectMapper(); CkanClient.configureObjectMapper(objectMapper); } @After public void tearDown() { objectMapper = null; } /** * todo we should do some ckan internal version detector (sic) */ @Test @Parameters(method = "clients") public void testApiVersionSupported(CkanClient client) { int version = client.getApiVersion(); assertTrue( "Found api version " + version + ", supported versions are: " + CkanClient.SUPPORTED_API_VERSIONS, CkanClient.SUPPORTED_API_VERSIONS.contains(version)); } @Test @Parameters(method = "clients") public void testDatasetList(CkanClient client) { List<String> dsl = client.getDatasetList(); assertTrue(dsl.size() > 0); } /** * Ckan docs don't tell offset starts with 0 * * For some weird reason {@link #DATI_MATERA} claims to have api v3 but does * not support limit & offset params */ @Test @Parameters(method = "clients") public void testDatasetListWithLimit(CkanClient client) { List<String> dsl = client.getDatasetList(1, 0); assertEquals(1, dsl.size()); } @Test @Parameters(method = "clients") public void testSearchDatasetsByText(CkanClient client) { List<String> dsl = client.getDatasetList(1, 0); assertTrue(dsl.size() > 0); SearchResults<CkanDataset> r = client.searchDatasets(dsl.get(0), 10, 0); assertTrue("I should get at least one result", r.getResults().size() > 0); } @Test @Parameters(method = "clients") public void testDatasetAndResource(CkanClient client) { List<String> dsl = client.getDatasetList(TEST_ELEMENTS, 0); assertTrue(dsl.size() > 0); List<FailedResourceException> failedResources = new ArrayList(); for (String datasetName : dsl.subList(0, Math.min(dsl.size(), TEST_ELEMENTS))) { CkanDataset dataset = client.getDataset(datasetName); assertEquals(datasetName, dataset.getName()); for (CkanResource resource : dataset.getResources().subList(0, Math.min(dataset.getResources().size(), TEST_ELEMENTS))) { try { client.getResource(resource.getId()); } catch (Exception ex) { failedResources.add(new FailedResourceException(client, "Error while fetching resource!", datasetName, resource.getId(), ex)); } } } if (!failedResources.isEmpty()) { throw new RuntimeException("Couldn't fetch these resources: \n " + failedResources.toString()); } } @Test @Parameters(method = "clients") public void testLicenseList(CkanClient client) { List<CkanLicense> licenses = client.getLicenseList(); assertTrue(licenses.size() > 0); } @Test @Parameters(method = "clients") public void testTagList(CkanClient client) { List<CkanTag> tl = client.getTagList(); assertTrue(tl.size() > 0); } /** * Searched by name */ @Test @Parameters(method = "clients") public void testTagNameList(CkanClient client) { List<CkanTag> tagList = client.getTagList(); assertTrue(tagList.size() > 0); String firstTagName = tagList.get(0).getName(); assertTrue(firstTagName.length() > 0); String searchText = firstTagName.substring(0, firstTagName.length() - 1); List<String> tl = client.getTagNamesList(searchText); assertTrue(tl.get(0).toLowerCase().contains(searchText)); } @Test @Parameters(method = "clients") public void testUserList(CkanClient client) { List<CkanUser> ul = client.getUserList(); assertTrue(ul.size() > 0); } /** * Tries to get some users. */ @Test @Parameters(method = "clients") public void testUser(CkanClient client) { List<CkanUser> ul = client.getUserList(); assertTrue(ul.size() > 0); for (CkanUser u : ul.subList(0, Math.min(ul.size(), TEST_ELEMENTS))) { CkanUser fetchedUser = client.getUser(u.getId()); assertEquals(u.getName(), fetchedUser.getName()); } } @Test @Parameters(method = "clients") public void testGroupList(CkanClient client) { List<CkanGroup> gl = client.getGroupList(); assertTrue(gl.size() > 0); // api actually returns the number of datasets, so we check they are properly 'converted'. assertEquals(null, gl.get(0).getPackages()); } @Test @Parameters(method = "clients") public void testGroupNames(CkanClient client) { List<String> gl = client.getGroupNames(); assertTrue(gl.size() > 0); } @Test @Parameters(method = "clients") public void testGroup(CkanClient client) { List<CkanGroup> gl = client.getGroupList(); assertTrue(gl.size() > 0); for (CkanGroup g : gl.subList(0, Math.min(gl.size(), TEST_ELEMENTS))) { CkanGroup fetchedGroup = client.getGroup(g.getId()); assertEquals(g.getName(), fetchedGroup.getName()); } } @Test @Parameters(method = "clients") public void testOrganizationList(CkanClient client) { List<CkanOrganization> gl = client.getOrganizationList(); assertTrue(gl.size() > 0); // api actually returns the number of datasets, so we check they are properly 'converted'. assertEquals(null, gl.get(0).getPackages()); } @Test @Parameters(method = "clients") public void testOrganizationNames(CkanClient client) { List<String> gl = client.getOrganizationNames(); assertTrue(gl.size() > 0); } @Test @Parameters(method = "clients") public void testOrganization(CkanClient client) { List<CkanOrganization> gl = client.getOrganizationList(); assertTrue(gl.size() > 0); for (CkanOrganization g : gl.subList(0, Math.min(gl.size(), TEST_ELEMENTS))) { CkanOrganization fetchedOrganization = client.getOrganization(g.getId()); assertEquals(g.getName(), fetchedOrganization.getName()); } } @Test @Parameters(method = "clients") public void testFormatList(CkanClient client) { Set<String> formats = client.getFormats(); assertTrue(formats.size() > 0); } @Test @Parameters(method = "clients") public void testSearchDatasetsByGroups(CkanClient client) { List<CkanGroup> groups = client.getGroupList(); assertTrue(groups.size() > 0); SearchResults<CkanDataset> r = client .searchDatasets(CkanQuery.filter().byGroupNames(groups.get(0).getName()), 10, 0); assertTrue("I should get at least one result", r.getResults().size() > 0); } @Test @Parameters(method = "clients") public void testSearchDatasetsByOrganization(CkanClient client) { List<CkanOrganization> organizations = client.getOrganizationList(); assertTrue(organizations.size() > 0); SearchResults<CkanDataset> r = client .searchDatasets(CkanQuery.filter().byOrganizationName(organizations.get(0).getName()), 10, 0); assertTrue("I should get at least one result", r.getResults().size() > 0); } @Test @Parameters(method = "clients") public void testSearchDatasetsByTags(CkanClient client) { List<String> datasetNamesList = client.getDatasetList(); assertTrue(datasetNamesList.size() > 0); List<CkanTag> tagList = client.getTagList(); assertTrue(tagList.size() > 0); String tagName; for (String datasetName : datasetNamesList.subList(0, Math.min(datasetList.size(), TEST_ELEMENTS))) { CkanDataset dataset = client.getDataset(datasetName); List<CkanTag> tags = dataset.getTags(); if (tags.size() > 0 && tags.get(0).getName().length() > 0) { tagName = tags.get(0).getName(); SearchResults<CkanDataset> r = client.searchDatasets(CkanQuery.filter().byTagNames(tagName), 10, 0); if (r.getResults().isEmpty()) { Assert.fail("I should find dataset " + dataset.getUrl() + " when searching for tag " + tagName); } else { return; } } } for (CkanTag tag : tagList.subList(0, Math.min(tagList.size(), TEST_ELEMENTS))) { SearchResults<CkanDataset> r = client.searchDatasets(CkanQuery.filter().byTagNames(tag.getName()), 10, 0); if (r.getResults().size() > 0) { return; } } Assert.fail("Couldn't find a dataset containing a tag so to be able to test search by tag."); } @Test @Parameters(method = "clients") public void testSearchDatasetsByLicenseIds(CkanClient client) throws JsonProcessingException { List<CkanLicense> licenses = client.getLicenseList(); assertTrue(licenses.size() > 0); for (CkanLicense license : licenses) { SearchResults<CkanDataset> r = client.searchDatasets(CkanQuery.filter().byLicenseId(license.getId()), 10, 0); if (r.getResults().size() > 0) { return; } } Assert.fail("I should get at least one dataset matching some license! Tried licenses were: " + objectMapper.writeValueAsString(licenses)); } @Test @Parameters(method = "clients") public void testCkanError(CkanClient client) { try { CkanDataset dataset = client.getDataset(UUID.randomUUID().toString()); fail(); } catch (CkanNotFoundException ex) { checkNotNull(ex.getCkanResponse().getError()); assertEquals(CkanError.NOT_FOUND_ERROR, ex.getCkanResponse().getError().getType()); } } @Test public void testFullSearch() { SearchResults<CkanDataset> r = new CkanClient(DATI_TRENTINO) .searchDatasets(CkanQuery.filter().byText("elenco dei prodotti trentini") .byGroupNames("agricoltura").byOrganizationName("pat-s-sviluppo-rurale") .byTagNames("prodotti tipici", "enogastronomia").byLicenseId("cc-zero"), 10, 0); assertEquals("cc-zero", r.getResults().get(0).getLicenseId()); assertTrue("I should get at least one result", r.getResults().size() > 0); } @Test public void testTimeout() { CkanClient myClient = CkanClient.builder().setCatalogUrl(UNREACHABLE_HOST).setTimeout(500).build(); try { myClient.getApiVersion(); } catch (CkanException ex) { } try { myClient.getDatasetList(); } catch (CkanException ex) { } } }