Java tutorial
/* * * Copyright 2013 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.ice.basic; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.netflix.ice.common.AwsUtils; import com.netflix.ice.common.Poller; import com.netflix.ice.common.TagGroup; import com.netflix.ice.processor.TagGroupWriter; import com.netflix.ice.reader.ReaderConfig; import com.netflix.ice.reader.TagGroupManager; import com.netflix.ice.reader.TagLists; import com.netflix.ice.tag.*; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.util.*; public class BasicTagGroupManager extends Poller implements TagGroupManager { private ReaderConfig config = ReaderConfig.getInstance(); private String dbName; private File file; private TreeMap<Long, Collection<TagGroup>> tagGroups; private TreeMap<Long, Collection<TagGroup>> tagGroupsWithResourceGroups; private Interval totalInterval; BasicTagGroupManager(Product product) { this.dbName = TagGroupWriter.DB_PREFIX + (product == null ? "all" : product.name); file = new File(config.localDir, dbName); try { poll(); } catch (Exception e) { logger.error("cannot poll data", e); } start(300); } @Override protected void poll() throws IOException { boolean downloaded = AwsUtils.downloadFileIfChanged(config.workS3BucketName, config.workS3BucketPrefix, file, 0); if (downloaded || tagGroups == null) { logger.info("trying to read from " + file); DataInputStream in = new DataInputStream(new FileInputStream(file)); try { TreeMap<Long, Collection<TagGroup>> tagGroupsWithResourceGroups = TagGroup.Serializer .deserializeTagGroups(config, in); TreeMap<Long, Collection<TagGroup>> tagGroups = removeResourceGroups(tagGroupsWithResourceGroups); Interval totalInterval = null; if (tagGroups.size() > 0) { totalInterval = new Interval(tagGroups.firstKey(), new DateTime(tagGroups.lastKey()).plusMonths(1).getMillis(), DateTimeZone.UTC); } this.totalInterval = totalInterval; this.tagGroups = tagGroups; this.tagGroupsWithResourceGroups = tagGroupsWithResourceGroups; logger.info("done reading " + file); } finally { in.close(); } } } @Override protected String getThreadName() { return this.dbName; } private TreeMap<Long, Collection<TagGroup>> removeResourceGroups( TreeMap<Long, Collection<TagGroup>> tagGroups) { TreeMap<Long, Collection<TagGroup>> result = Maps.newTreeMap(); for (Long key : tagGroups.keySet()) { Collection<TagGroup> from = tagGroups.get(key); Set<TagGroup> to = Sets.newHashSet(); for (TagGroup tagGroup : from) { if (tagGroup.resourceGroup != null) to.add(new TagGroup(tagGroup.account, tagGroup.region, tagGroup.zone, tagGroup.product, tagGroup.operation, tagGroup.usageType, null)); else to.add(tagGroup); } result.put(key, to); } return result; } private Set<TagGroup> getTagGroupsInRange(Collection<Long> monthMillis) { Set<TagGroup> tagGroupsInRange = Sets.newHashSet(); for (Long monthMilli : monthMillis) { tagGroupsInRange.addAll(this.tagGroups.get(monthMilli)); } return tagGroupsInRange; } private Set<TagGroup> getTagGroupsWithResourceGroupsInRange(Collection<Long> monthMillis) { Set<TagGroup> tagGroupsInRange = Sets.newHashSet(); for (Long monthMilli : monthMillis) { tagGroupsInRange.addAll(this.tagGroupsWithResourceGroups.get(monthMilli)); } return tagGroupsInRange; } private Collection<Long> getMonthMillis(Interval interval) { Set<Long> result = Sets.newTreeSet(); for (Long milli : tagGroups.keySet()) { DateTime monthDate = new DateTime(milli, DateTimeZone.UTC); if (new Interval(monthDate, monthDate.plusMonths(1)).overlap(interval) != null) result.add(milli); } return result; } public Collection<Account> getAccounts(Interval interval, TagLists tagLists) { Set<Account> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup)) result.add(tagGroup.account); } return result; } public Collection<Region> getRegions(Interval interval, TagLists tagLists) { Set<Region> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup)) result.add(tagGroup.region); } return result; } public Collection<Zone> getZones(Interval interval, TagLists tagLists) { Set<Zone> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup) && tagGroup.zone != null) result.add(tagGroup.zone); } return result; } public Collection<Product> getProducts(Interval interval, TagLists tagLists) { Set<Product> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup)) result.add(tagGroup.product); } return result; } public Collection<Operation> getOperations(Interval interval, TagLists tagLists) { Set<Operation> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagGroup.product == Product.emr || tagGroup.product == Product.redshift) { int iii = 0; } if (tagLists.contains(tagGroup)) result.add(tagGroup.operation); } return result; } public Collection<UsageType> getUsageTypes(Interval interval, TagLists tagLists) { Set<UsageType> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup)) result.add(tagGroup.usageType); } return result; } public Collection<ResourceGroup> getResourceGroups(Interval interval, TagLists tagLists) { Set<ResourceGroup> result = Sets.newTreeSet(); Set<TagGroup> tagGroupsInRange = getTagGroupsWithResourceGroupsInRange(getMonthMillis(interval)); for (TagGroup tagGroup : tagGroupsInRange) { if (tagLists.contains(tagGroup) && tagGroup.resourceGroup != null) result.add(tagGroup.resourceGroup); } return result; } public Collection<Account> getAccounts(TagLists tagLists) { return this.getAccounts(totalInterval, tagLists); } public Collection<Region> getRegions(TagLists tagLists) { return this.getRegions(totalInterval, tagLists); } public Collection<Zone> getZones(TagLists tagLists) { return this.getZones(totalInterval, tagLists); } public Collection<Product> getProducts(TagLists tagLists) { return this.getProducts(totalInterval, tagLists); } public Collection<Operation> getOperations(TagLists tagLists) { return this.getOperations(totalInterval, tagLists); } public Collection<UsageType> getUsageTypes(TagLists tagLists) { return this.getUsageTypes(totalInterval, tagLists); } public Collection<ResourceGroup> getResourceGroups(TagLists tagLists) { return this.getResourceGroups(totalInterval, tagLists); } public Interval getOverlapInterval(Interval interval) { return totalInterval == null ? null : totalInterval.overlap(interval); } public Map<Tag, TagLists> getTagListsMap(Interval interval, TagLists tagLists, TagType groupBy, boolean forReservation) { Map<Tag, TagLists> result = Maps.newHashMap(); TagLists tagListsForTag = tagLists; if (groupBy != TagType.ResourceGroup && tagLists.resourceGroups != null && tagLists.resourceGroups.size() > 0) { tagListsForTag = new TagLists(tagLists.accounts, tagLists.regions, tagLists.zones, tagLists.products, tagLists.operations, tagLists.usageTypes); } List<Tag> groupByTags = Lists.newArrayList(); switch (groupBy) { case Account: groupByTags.addAll(getAccounts(interval, tagListsForTag)); break; case Region: groupByTags.addAll(getRegions(interval, tagListsForTag)); break; case Zone: groupByTags.addAll(getZones(interval, tagListsForTag)); break; case Product: groupByTags.addAll(getProducts(interval, tagListsForTag)); break; case Operation: groupByTags.addAll(getOperations(interval, tagListsForTag)); break; case UsageType: groupByTags.addAll(getUsageTypes(interval, tagListsForTag)); break; case ResourceGroup: groupByTags.addAll(getResourceGroups(interval, tagListsForTag)); break; } if (groupBy == TagType.Operation && !forReservation) { for (Operation.ReservationOperation lentOp : Operation.getLentInstances()) groupByTags.remove(lentOp); } for (Tag tag : groupByTags) { if (tagLists.contains(tag, groupBy)) { TagLists tmp = tagLists.getTagLists(tag, groupBy); if (!forReservation && groupBy != TagType.Operation) { if (tmp.operations == null || tmp.operations.size() == 0) { List<Operation> operations = Lists.newArrayList(getOperations(tmp)); tmp = new TagLists(tmp.accounts, tmp.regions, tmp.zones, tmp.products, operations, tmp.usageTypes, tmp.resourceGroups); } for (Operation.ReservationOperation lentOp : Operation.getLentInstances()) tmp.operations.remove(lentOp); } result.put(tag, tmp); } } return result; } }