public class NodeQueueLoadMonitor extends Object implements ClusterMonitor
Modifier and Type | Class and Description |
---|---|
static class |
NodeQueueLoadMonitor.LoadComparator
The comparator used to specify the metric against which the load
of two Nodes are compared.
|
Modifier and Type | Field and Description |
---|---|
protected Map<org.apache.hadoop.yarn.api.records.NodeId,ClusterNode> |
clusterNodes |
protected ReentrantReadWriteLock |
clusterNodesLock |
protected NodeQueueLoadMonitor.LoadComparator |
comparator |
protected static org.slf4j.Logger |
LOG |
protected Map<String,RMNode> |
nodeByHostName |
protected Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>> |
nodeIdsByRack |
protected int |
numNodesForAnyAllocation |
protected List<org.apache.hadoop.yarn.api.records.NodeId> |
sortedNodes |
protected ReentrantReadWriteLock |
sortedNodesLock |
protected QueueLimitCalculator |
thresholdCalculator |
Constructor and Description |
---|
NodeQueueLoadMonitor(long nodeComputationInterval,
NodeQueueLoadMonitor.LoadComparator comparator,
int numNodes) |
Modifier and Type | Method and Description |
---|---|
protected void |
addIntoNodeIdsByRack(RMNode addedNode) |
void |
addNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerStatuses,
RMNode rmNode) |
protected List<org.apache.hadoop.yarn.api.records.NodeId> |
getCandidatesForSelectAnyNode() |
QueueLimitCalculator |
getThresholdCalculator() |
void |
initThresholdCalculator(float sigma,
int limitMin,
int limitMax) |
protected void |
onExistingNodeUpdated(RMNode rmNode,
ClusterNode clusterNode,
org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status) |
protected void |
onNewNodeAdded(RMNode rmNode,
org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status) |
protected void |
onNodeRemoved(ClusterNode node)
Provide an integration point for extended class
|
protected void |
removeFromNodeIdsByRack(RMNode removedNode) |
void |
removeNode(RMNode removedRMNode) |
RMNode |
selectAnyNode(Set<String> blacklist,
org.apache.hadoop.yarn.api.records.Resource request)
Selects a node from all ClusterNodes for resource allocation,
excluding blacklisted nodes.
|
List<org.apache.hadoop.yarn.api.records.NodeId> |
selectLeastLoadedNodes(int k)
Returns 'K' of the least Loaded Node Ids as ordered list.
|
RMNode |
selectLocalNode(String hostName,
Set<String> blacklist,
org.apache.hadoop.yarn.api.records.Resource request)
Selects the node as specified by hostName for resource allocation,
unless the node has been blacklisted.
|
List<org.apache.hadoop.yarn.api.records.NodeId> |
selectNodes()
Returns all Node Ids as ordered list from Least to Most Loaded.
|
RMNode |
selectRackLocalNode(String rackName,
Set<String> blacklist,
org.apache.hadoop.yarn.api.records.Resource request)
Selects a node from the rack as specified by rackName
for resource allocation, excluding blacklisted nodes
|
protected List<ClusterNode> |
sortNodes(boolean excludeFullNodes) |
void |
stop() |
void |
updateNode(RMNode rmNode) |
void |
updateNodeResource(RMNode rmNode,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption) |
protected void |
updateSortedNodes() |
protected static final org.slf4j.Logger LOG
protected int numNodesForAnyAllocation
protected final List<org.apache.hadoop.yarn.api.records.NodeId> sortedNodes
protected final Map<org.apache.hadoop.yarn.api.records.NodeId,ClusterNode> clusterNodes
protected final Map<String,Set<org.apache.hadoop.yarn.api.records.NodeId>> nodeIdsByRack
protected final NodeQueueLoadMonitor.LoadComparator comparator
protected QueueLimitCalculator thresholdCalculator
protected ReentrantReadWriteLock sortedNodesLock
protected ReentrantReadWriteLock clusterNodesLock
public NodeQueueLoadMonitor(long nodeComputationInterval, NodeQueueLoadMonitor.LoadComparator comparator, int numNodes)
protected void updateSortedNodes()
public QueueLimitCalculator getThresholdCalculator()
public void stop()
public void initThresholdCalculator(float sigma, int limitMin, int limitMax)
public void addNode(List<org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus> containerStatuses, RMNode rmNode)
addNode
in interface ClusterMonitor
public void removeNode(RMNode removedRMNode)
removeNode
in interface ClusterMonitor
protected void onNodeRemoved(ClusterNode node)
node
- the node removedpublic void updateNode(RMNode rmNode)
updateNode
in interface ClusterMonitor
protected void onNewNodeAdded(RMNode rmNode, org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)
protected void onExistingNodeUpdated(RMNode rmNode, ClusterNode clusterNode, org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus status)
public void updateNodeResource(RMNode rmNode, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource
in interface ClusterMonitor
public List<org.apache.hadoop.yarn.api.records.NodeId> selectNodes()
public List<org.apache.hadoop.yarn.api.records.NodeId> selectLeastLoadedNodes(int k)
k
- max number of nodes to returnpublic RMNode selectLocalNode(String hostName, Set<String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)
hostName
- the hostname of the node for local resource allocationblacklist
- the blacklisted nodesrequest
- the requested resourcepublic RMNode selectRackLocalNode(String rackName, Set<String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)
rackName
- the rack name for rack-local resource allocationblacklist
- the blacklisted nodesrequest
- the requested resourcepublic RMNode selectAnyNode(Set<String> blacklist, org.apache.hadoop.yarn.api.records.Resource request)
blacklist
- the blacklisted nodesrequest
- the requested resourceprotected List<org.apache.hadoop.yarn.api.records.NodeId> getCandidatesForSelectAnyNode()
protected void removeFromNodeIdsByRack(RMNode removedNode)
protected void addIntoNodeIdsByRack(RMNode addedNode)
protected List<ClusterNode> sortNodes(boolean excludeFullNodes)
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.