Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.tajo.master; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.util.RackResolver; import org.apache.tajo.util.NetUtils; import java.util.*; import java.util.Map.Entry; /** * DefaultFragmentScheduleAlgorithm selects a fragment randomly for the given argument. * For example, when getHostLocalFragment(host, disk) is called, this algorithm randomly selects a fragment among * the fragments which are stored at the disk of the host specified by the arguments. */ public class DefaultFragmentScheduleAlgorithm implements FragmentScheduleAlgorithm { private final static Log LOG = LogFactory.getLog(DefaultFragmentScheduleAlgorithm.class); private Map<String, Map<Integer, FragmentsPerDisk>> fragmentHostMapping = new HashMap<String, Map<Integer, FragmentsPerDisk>>(); private Map<String, Set<FragmentPair>> rackFragmentMapping = new HashMap<String, Set<FragmentPair>>(); private int fragmentNum = 0; private Random random = new Random(System.currentTimeMillis()); public static class FragmentsPerDisk { private Integer diskId; private Set<FragmentPair> fragmentPairSet; public FragmentsPerDisk(Integer diskId) { this.diskId = diskId; this.fragmentPairSet = Collections.newSetFromMap(new HashMap<FragmentPair, Boolean>()); } public Integer getDiskId() { return diskId; } public Set<FragmentPair> getFragmentPairSet() { return fragmentPairSet; } public void addFragmentPair(FragmentPair fragmentPair) { fragmentPairSet.add(fragmentPair); } public boolean removeFragmentPair(FragmentPair fragmentPair) { return fragmentPairSet.remove(fragmentPair); } public int size() { return fragmentPairSet.size(); } public Iterator<FragmentPair> getFragmentPairIterator() { return fragmentPairSet.iterator(); } public boolean isEmpty() { return fragmentPairSet.isEmpty(); } } @Override public void addFragment(FragmentPair fragmentPair) { String[] hosts = fragmentPair.getLeftFragment().getHosts(); int[] diskIds = fragmentPair.getLeftFragment().getDiskIds(); for (int i = 0; i < hosts.length; i++) { addFragment(hosts[i], diskIds[i], fragmentPair); } fragmentNum++; } private void addFragment(String host, Integer diskId, FragmentPair fragmentPair) { // update the fragment maps per host String normalizeHost = NetUtils.normalizeHost(host); Map<Integer, FragmentsPerDisk> diskFragmentMap; if (fragmentHostMapping.containsKey(normalizeHost)) { diskFragmentMap = fragmentHostMapping.get(normalizeHost); } else { diskFragmentMap = new HashMap<Integer, FragmentsPerDisk>(); fragmentHostMapping.put(normalizeHost, diskFragmentMap); } FragmentsPerDisk fragmentsPerDisk; if (diskFragmentMap.containsKey(diskId)) { fragmentsPerDisk = diskFragmentMap.get(diskId); } else { fragmentsPerDisk = new FragmentsPerDisk(diskId); diskFragmentMap.put(diskId, fragmentsPerDisk); } fragmentsPerDisk.addFragmentPair(fragmentPair); // update the fragment maps per rack String rack = RackResolver.resolve(normalizeHost).getNetworkLocation(); Set<FragmentPair> fragmentPairList; if (rackFragmentMapping.containsKey(rack)) { fragmentPairList = rackFragmentMapping.get(rack); } else { fragmentPairList = Collections.newSetFromMap(new HashMap<FragmentPair, Boolean>()); rackFragmentMapping.put(rack, fragmentPairList); } fragmentPairList.add(fragmentPair); } @Override public void removeFragment(FragmentPair fragmentPair) { boolean removed = false; for (String eachHost : fragmentPair.getLeftFragment().getHosts()) { String normalizedHost = NetUtils.normalizeHost(eachHost); Map<Integer, FragmentsPerDisk> diskFragmentMap = fragmentHostMapping.get(normalizedHost); for (Entry<Integer, FragmentsPerDisk> entry : diskFragmentMap.entrySet()) { FragmentsPerDisk fragmentsPerDisk = entry.getValue(); removed = fragmentsPerDisk.removeFragmentPair(fragmentPair); if (removed) { if (fragmentsPerDisk.size() == 0) { diskFragmentMap.remove(entry.getKey()); } if (diskFragmentMap.size() == 0) { fragmentHostMapping.remove(normalizedHost); } break; } } String rack = RackResolver.resolve(normalizedHost).getNetworkLocation(); if (rackFragmentMapping.containsKey(rack)) { Set<FragmentPair> fragmentPairs = rackFragmentMapping.get(rack); fragmentPairs.remove(fragmentPair); if (fragmentPairs.size() == 0) { rackFragmentMapping.remove(rack); } } } if (removed) { fragmentNum--; } } /** * Randomly select a fragment among the fragments stored on the host. * @param host * @return a randomly selected fragment */ @Override public FragmentPair getHostLocalFragment(String host) { String normalizedHost = NetUtils.normalizeHost(host); if (fragmentHostMapping.containsKey(normalizedHost)) { Collection<FragmentsPerDisk> disks = fragmentHostMapping.get(normalizedHost).values(); Iterator<FragmentsPerDisk> diskIterator = disks.iterator(); int randomIndex = random.nextInt(disks.size()); FragmentsPerDisk fragmentsPerDisk = null; for (int i = 0; i < randomIndex; i++) { fragmentsPerDisk = diskIterator.next(); } if (fragmentsPerDisk != null) { Iterator<FragmentPair> fragmentIterator = fragmentsPerDisk.getFragmentPairIterator(); if (fragmentIterator.hasNext()) { return fragmentIterator.next(); } } } return null; } /** * Randomly select a fragment among the fragments stored at the disk of the host. * @param host * @param diskId * @return a randomly selected fragment */ @Override public FragmentPair getHostLocalFragment(String host, Integer diskId) { String normalizedHost = NetUtils.normalizeHost(host); if (fragmentHostMapping.containsKey(normalizedHost)) { Map<Integer, FragmentsPerDisk> fragmentsPerDiskMap = fragmentHostMapping.get(normalizedHost); if (fragmentsPerDiskMap.containsKey(diskId)) { FragmentsPerDisk fragmentsPerDisk = fragmentsPerDiskMap.get(diskId); if (!fragmentsPerDisk.isEmpty()) { return fragmentsPerDisk.getFragmentPairIterator().next(); } } } return null; } /** * Randomly select a fragment among the fragments stored on nodes of the same rack with the host. * @param host * @return a randomly selected fragment */ @Override public FragmentPair getRackLocalFragment(String host) { String rack = RackResolver.resolve(host).getNetworkLocation(); if (rackFragmentMapping.containsKey(rack)) { Set<FragmentPair> fragmentPairs = rackFragmentMapping.get(rack); if (!fragmentPairs.isEmpty()) { return fragmentPairs.iterator().next(); } } return null; } /** * Randomly select a fragment among the total fragments. * @return a randomly selected fragment */ @Override public FragmentPair getRandomFragment() { if (!fragmentHostMapping.isEmpty()) { return fragmentHostMapping.values().iterator().next().values().iterator().next() .getFragmentPairIterator().next(); } return null; } @Override public FragmentPair[] getAllFragments() { List<FragmentPair> fragmentPairs = new ArrayList<FragmentPair>(); for (Map<Integer, FragmentsPerDisk> eachDiskFragmentMap : fragmentHostMapping.values()) { for (FragmentsPerDisk fragmentsPerDisk : eachDiskFragmentMap.values()) { fragmentPairs.addAll(fragmentsPerDisk.fragmentPairSet); } } return fragmentPairs.toArray(new FragmentPair[fragmentPairs.size()]); } @Override public int size() { return fragmentNum; } }