public abstract class AbstractCSQueue extends Object implements CSQueue
CSQueue
methods common for every queue class in Capacity
Scheduler.Modifier and Type | Class and Description |
---|---|
static class |
AbstractCSQueue.CapacityConfigType |
Modifier and Type | Field and Description |
---|---|
protected ActivitiesManager |
activitiesManager |
protected org.apache.hadoop.yarn.security.YarnAuthorizationProvider |
authorizer |
protected AbstractCSQueue.CapacityConfigType |
capacityConfigType |
protected Map<String,QueueCapacityVector> |
configuredCapacityVectors |
protected Map<String,QueueCapacityVector> |
configuredMaxCapacityVectors |
protected QueueAllocationSettings |
queueAllocationSettings |
protected CapacitySchedulerQueueContext |
queueContext |
protected org.apache.hadoop.yarn.security.PrivilegedEntity |
queueEntity |
protected QueueNodeLabelsSettings |
queueNodeLabelsSettings |
protected QueuePath |
queuePath |
protected ReentrantReadWriteLock.ReadLock |
readLock |
protected ReentrantReadWriteLock.WriteLock |
writeLock |
Constructor and Description |
---|
AbstractCSQueue(CapacitySchedulerQueueContext queueContext,
String queueName,
CSQueue parent,
CSQueue old) |
Modifier and Type | Method and Description |
---|---|
boolean |
accept(org.apache.hadoop.yarn.api.records.Resource cluster,
ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request)
Checks whether this queue could accept the container allocation request.
|
void |
activateQueue()
Sets the state of this queue to RUNNING.
|
protected void |
appFinished()
Stops this queue if no application is currently running on the queue.
|
CSAssignment |
assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource,
FiCaSchedulerNode node,
ResourceLimits resourceLimits,
SchedulingMode schedulingMode) |
protected boolean |
checkConfigTypeIsAbsoluteResource(String queuePath,
String label) |
void |
decPendingResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec)
When an outstanding resource is fulfilled or canceled, calling this will
decrease pending resource in a queue.
|
void |
decReservedResource(String partition,
org.apache.hadoop.yarn.api.records.Resource reservedRes)
Decrement Reserved Capacity
|
void |
decUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToDec,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
float |
getAbsoluteCapacity()
Get capacity of the parent of the queue as a function of the
cumulative capacity in the cluster.
|
float |
getAbsoluteMaximumCapacity()
Get maximum-capacity of the queue as a funciton of the cumulative capacity
of the cluster.
|
float |
getAbsoluteUsedCapacity()
Get the current absolute used capacity of the queue
relative to the entire cluster.
|
Set<String> |
getAccessibleNodeLabels()
Get labels can be accessed of this queue
labels={*}, means this queue can access any label
labels={ }, means this queue cannot access any label except node without label
labels={a, b, c} means this queue can access a or b or c
|
Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> |
getACLs() |
float |
getCapacity()
Get the configured capacity of the queue.
|
AbstractCSQueue.CapacityConfigType |
getCapacityConfigType()
Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE.
|
protected String |
getCapacityOrWeightString() |
QueueCapacityVector |
getConfiguredCapacityVector(String label)
Get configured capacity vector parsed from the capacity config
of the queue.
|
QueueCapacityVector |
getConfiguredMaxCapacityVector(String label)
Get configured maximum capacity vector parsed from the capacity config
of the queue.
|
Set<String> |
getConfiguredNodeLabels() |
long |
getDefaultApplicationLifetime()
Get the default lifetime in seconds of an application which is submitted to
this queue.
|
org.apache.hadoop.yarn.api.records.Priority |
getDefaultApplicationPriority()
Get the Default Application Priority for this queue
|
boolean |
getDefaultAppLifetimeWasSpecifiedInConfig()
Get the indicator of whether or not the default application lifetime was
set by a config property or was calculated by the capacity scheduler.
|
String |
getDefaultNodeLabelExpression()
Get default label expression of this queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getEffectiveCapacity(String label)
Get effective capacity of queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getEffectiveCapacityDown(String label,
org.apache.hadoop.yarn.api.records.Resource factor)
Get effective capacity of queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getEffectiveMaxCapacity(String label)
Get effective max capacity of queue.
|
org.apache.hadoop.yarn.api.records.Resource |
getEffectiveMaxCapacityDown(String label,
org.apache.hadoop.yarn.api.records.Resource factor)
Get effective max capacity of queue.
|
boolean |
getIntraQueuePreemptionDisabled()
Check whether intra-queue preemption is disabled for this queue
|
boolean |
getIntraQueuePreemptionDisabledInHierarchy()
Determines whether or not the intra-queue preemption disabled switch is set
at any level in this queue's hierarchy.
|
Iterator<RMContainer> |
getKillableContainers(String partition) |
protected org.apache.hadoop.yarn.api.records.Resource |
getMaximumAbsoluteResource(String queuePath,
String label) |
org.apache.hadoop.yarn.api.records.Resource |
getMaximumAllocation() |
long |
getMaximumApplicationLifetime()
Get the maximum lifetime in seconds of an application which is submitted to
this queue.
|
float |
getMaximumCapacity()
Get the configured maximum-capacity of the queue.
|
int |
getMaxParallelApps()
Get the max-parallel-applications property of the queue
|
CSQueueMetrics |
getMetrics()
Get the queue metrics
|
protected org.apache.hadoop.yarn.api.records.Resource |
getMinimumAbsoluteResource(String queuePath,
String label) |
org.apache.hadoop.yarn.api.records.Resource |
getMinimumAllocation() |
String |
getMultiNodeSortingPolicyName()
Get Multi Node scheduling policy name.
|
Set<String> |
getNodeLabelsForQueue()
Returns the union of all node labels that could be accessed by this queue based on accessible
node labels and configured node labels properties.
|
int |
getNumContainers() |
CSQueue |
getParent()
Get the parent
Queue . |
boolean |
getPreemptionDisabled()
Check whether disable_preemption property is set for this queue
|
org.apache.hadoop.yarn.api.records.Priority |
getPriority()
Get priority of queue
|
org.apache.hadoop.yarn.security.PrivilegedEntity |
getPrivilegedEntity() |
QueueCapacities |
getQueueCapacities()
Get QueueCapacities of this queue
|
CapacitySchedulerQueueContext |
getQueueContext() |
protected org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo() |
String |
getQueueName()
Get the queue's internal reference name.
|
String |
getQueuePath()
Get the full name of the queue, including the heirarchy.
|
QueuePath |
getQueuePathObject()
Gets the queue path object.
|
QueueResourceQuotas |
getQueueResourceQuotas()
Get QueueResourceQuotas associated with each queue.
|
ResourceUsage |
getQueueResourceUsage()
Get ResourceUsage of this queue
|
String |
getQueueShortName()
Get the queue's legacy name.
|
ReentrantReadWriteLock.ReadLock |
getReadLock()
Get readLock associated with the Queue.
|
org.apache.hadoop.yarn.api.records.QueueState |
getState()
Get the current run-state of the queue
|
org.apache.hadoop.yarn.api.records.Resource |
getTotalKillableResource(String partition) |
float |
getUsedCapacity()
Get the current used capacity of nodes without label(s) of the queue
and it's children (if any).
|
org.apache.hadoop.yarn.api.records.Resource |
getUsedResources()
Get the currently utilized resources which allocated at nodes without any
labels in the cluster by the queue and children (if any).
|
UserWeights |
getUserWeights()
Get the UserWeights object that wraps a map of usernames and weights
|
ReentrantReadWriteLock.WriteLock |
getWriteLock()
Get writeLock associated with the Queue.
|
boolean |
hasAccess(org.apache.hadoop.yarn.api.records.QueueACL acl,
org.apache.hadoop.security.UserGroupInformation user)
Checks whether the user has the required permission to execute the action of
QueueACL . |
void |
incPendingResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc)
When new outstanding resource is asked, calling this will increase pending
resource in a queue.
|
void |
incReservedResource(String partition,
org.apache.hadoop.yarn.api.records.Resource reservedRes)
Increment Reserved Capacity
|
void |
incUsedResource(String nodeLabel,
org.apache.hadoop.yarn.api.records.Resource resourceToInc,
SchedulerApplicationAttempt application)
When partition of node updated, we will update queue's resource usage if it
has container(s) running on that.
|
boolean |
isDynamicQueue()
Checks whether the queue is a dynamic queue (created dynamically in the fashion of auto queue
creation v2).
|
boolean |
isEligibleForAutoDeletion()
Checks whether this queue is a dynamic queue and could be deleted.
|
boolean |
isInactiveDynamicQueue()
Checks whether this queue is a dynamic queue and there has not been an application submission
on it for a configured period of time.
|
boolean |
isReservationsContinueLooking()
Returns whether we should continue to look at all heart beating nodes even
after the reservation limit was hit.
|
protected void |
overrideCapacityVectorsForSpecialQueues(String label) |
protected void |
parseAndSetDynamicTemplates()
Set properties specific to dynamic queues.
|
void |
recoverDrainingState()
Recursively sets the state of this queue and the state of its parent to DRAINING.
|
protected void |
releaseResource(org.apache.hadoop.yarn.api.records.Resource clusterResource,
org.apache.hadoop.yarn.api.records.Resource resource,
String nodePartition)
Decrements resource usage of the queue and all related statistics and metrics that depends on
it.
|
void |
setConfiguredMaxCapacityVector(String label,
QueueCapacityVector maxCapacityVector)
Sets the configured maximum capacity vector to a specific value.
|
void |
setConfiguredMinCapacityVector(String label,
QueueCapacityVector minCapacityVector)
Sets the configured minimum capacity vector to a specific value.
|
void |
setDynamicQueue(boolean dynamicQueue) |
protected void |
setDynamicQueueACLProperties() |
void |
setMaxParallelApps(int maxParallelApps) |
void |
setMultiNodeSortingPolicyName(String policyName) |
void |
setParent(CSQueue newParentQueue)
Set the parent
Queue . |
protected void |
setupConfigurableCapacities()
Sets up capacity and weight values from configuration.
|
protected void |
setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource)
Initialize queue properties that are based on configuration.
|
protected void |
updateAbsoluteCapacities() |
protected void |
updateCapacityConfigType() |
protected void |
updateConfigurableResourceLimits(org.apache.hadoop.yarn.api.records.Resource clusterResource)
Initializes configured minimum and maximum capacity from configuration, if capacity is defined
in ABSOLUTE node.
|
void |
updateQueueState(org.apache.hadoop.yarn.api.records.QueueState queueState)
Update the queue state.
|
void |
validateSubmitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String userName,
String queue)
Validate submitApplication api so that moveApplication do a pre-check.
|
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
apply, assignContainers, attachContainer, collectSchedulerApplications, completedContainer, detachContainer, finishApplication, finishApplicationAttempt, getAbstractUsersManager, getChildQueues, getChildQueuesByTryLock, getNumApplications, refreshAfterResourceCalculation, reinitialize, submitApplication, submitApplicationAttempt, submitApplicationAttempt, updateClusterResource
stopQueue
getQueueInfo, getQueueUserAclInfo, recoverContainer
protected final QueueAllocationSettings queueAllocationSettings
protected final QueuePath queuePath
protected QueueNodeLabelsSettings queueNodeLabelsSettings
protected final org.apache.hadoop.yarn.security.PrivilegedEntity queueEntity
protected AbstractCSQueue.CapacityConfigType capacityConfigType
protected Map<String,QueueCapacityVector> configuredCapacityVectors
protected Map<String,QueueCapacityVector> configuredMaxCapacityVectors
protected CapacitySchedulerQueueContext queueContext
protected org.apache.hadoop.yarn.security.YarnAuthorizationProvider authorizer
protected ActivitiesManager activitiesManager
protected ReentrantReadWriteLock.ReadLock readLock
protected ReentrantReadWriteLock.WriteLock writeLock
public AbstractCSQueue(CapacitySchedulerQueueContext queueContext, String queueName, CSQueue parent, CSQueue old)
protected void setupConfigurableCapacities()
public String getQueuePath()
CSQueue
getQueuePath
in interface CSQueue
public QueuePath getQueuePathObject()
CSQueue
getQueuePathObject
in interface CSQueue
public float getCapacity()
CSQueue
getCapacity
in interface CSQueue
public float getAbsoluteCapacity()
CSQueue
getAbsoluteCapacity
in interface CSQueue
public float getAbsoluteMaximumCapacity()
CSQueue
getAbsoluteMaximumCapacity
in interface CSQueue
public float getAbsoluteUsedCapacity()
CSQueue
getAbsoluteUsedCapacity
in interface CSQueue
public float getMaximumCapacity()
CSQueue
getMaximumCapacity
in interface CSQueue
public float getUsedCapacity()
CSQueue
getUsedCapacity
in interface CSQueue
public org.apache.hadoop.yarn.api.records.Resource getUsedResources()
CSQueue
getUsedResources
in interface CSQueue
public int getNumContainers()
public org.apache.hadoop.yarn.api.records.QueueState getState()
CSQueue
getState
in interface CSQueue
getState
in interface SchedulerQueue<CSQueue>
public CSQueueMetrics getMetrics()
Queue
getMetrics
in interface Queue
public String getQueueShortName()
CSQueue
getQueueShortName
in interface CSQueue
public String getQueueName()
CSQueue
getQueueName
in interface CSQueue
getQueueName
in interface Queue
public CSQueue getParent()
CSQueue
Queue
.getParent
in interface CSQueue
getParent
in interface SchedulerQueue<CSQueue>
public void setParent(CSQueue newParentQueue)
CSQueue
Queue
.public org.apache.hadoop.yarn.security.PrivilegedEntity getPrivilegedEntity()
getPrivilegedEntity
in interface CSQueue
public CapacitySchedulerQueueContext getQueueContext()
public Set<String> getAccessibleNodeLabels()
Queue
getAccessibleNodeLabels
in interface Queue
public boolean hasAccess(org.apache.hadoop.yarn.api.records.QueueACL acl, org.apache.hadoop.security.UserGroupInformation user)
QueueACL
.public String getDefaultNodeLabelExpression()
Queue
getDefaultNodeLabelExpression
in interface Queue
protected void setupQueueConfigs(org.apache.hadoop.yarn.api.records.Resource clusterResource) throws IOException
clusterResource
- overall resource of the clusterIOException
- if configuration is set in a way that is inconsistentprotected void parseAndSetDynamicTemplates()
protected void setDynamicQueueACLProperties()
protected void overrideCapacityVectorsForSpecialQueues(String label)
protected org.apache.hadoop.yarn.api.records.Resource getMinimumAbsoluteResource(String queuePath, String label)
protected org.apache.hadoop.yarn.api.records.Resource getMaximumAbsoluteResource(String queuePath, String label)
protected boolean checkConfigTypeIsAbsoluteResource(String queuePath, String label)
protected void updateCapacityConfigType()
protected void updateConfigurableResourceLimits(org.apache.hadoop.yarn.api.records.Resource clusterResource)
clusterResource
- overall resource of the clusterpublic AbstractCSQueue.CapacityConfigType getCapacityConfigType()
CSQueue
getCapacityConfigType
in interface CSQueue
public org.apache.hadoop.yarn.api.records.Resource getEffectiveCapacity(String label)
CSQueue
getEffectiveCapacity
in interface CSQueue
label
- partitionpublic org.apache.hadoop.yarn.api.records.Resource getEffectiveCapacityDown(String label, org.apache.hadoop.yarn.api.records.Resource factor)
CSQueue
getEffectiveCapacityDown
in interface CSQueue
label
- partitionfactor
- factor to normalize downpublic org.apache.hadoop.yarn.api.records.Resource getEffectiveMaxCapacity(String label)
CSQueue
getEffectiveMaxCapacity
in interface CSQueue
label
- partitionpublic org.apache.hadoop.yarn.api.records.Resource getEffectiveMaxCapacityDown(String label, org.apache.hadoop.yarn.api.records.Resource factor)
CSQueue
getEffectiveMaxCapacityDown
in interface CSQueue
label
- partitionfactor
- factor to normalize downpublic QueueCapacityVector getConfiguredCapacityVector(String label)
CSQueue
getConfiguredCapacityVector
in interface CSQueue
label
- node label (partition)public QueueCapacityVector getConfiguredMaxCapacityVector(String label)
CSQueue
getConfiguredMaxCapacityVector
in interface CSQueue
label
- node label (partition)public void setConfiguredMinCapacityVector(String label, QueueCapacityVector minCapacityVector)
CSQueue
setConfiguredMinCapacityVector
in interface CSQueue
label
- node label (partition)minCapacityVector
- capacity vectorpublic void setConfiguredMaxCapacityVector(String label, QueueCapacityVector maxCapacityVector)
CSQueue
setConfiguredMaxCapacityVector
in interface CSQueue
label
- node label (partition)maxCapacityVector
- capacity vectorprotected org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo()
@InterfaceAudience.Private public org.apache.hadoop.yarn.api.records.Resource getMaximumAllocation()
getMaximumAllocation
in interface CSQueue
@InterfaceAudience.Private public org.apache.hadoop.yarn.api.records.Resource getMinimumAllocation()
getMinimumAllocation
in interface CSQueue
protected void releaseResource(org.apache.hadoop.yarn.api.records.Resource clusterResource, org.apache.hadoop.yarn.api.records.Resource resource, String nodePartition)
clusterResource
- overall cluster resourceresource
- resource amount to decrementnodePartition
- node label@InterfaceAudience.Private public boolean isReservationsContinueLooking()
@InterfaceAudience.Private public Map<org.apache.hadoop.yarn.security.AccessType,org.apache.hadoop.security.authorize.AccessControlList> getACLs()
@InterfaceAudience.Private public boolean getPreemptionDisabled()
CSQueue
getPreemptionDisabled
in interface CSQueue
@InterfaceAudience.Private public boolean getIntraQueuePreemptionDisabled()
CSQueue
getIntraQueuePreemptionDisabled
in interface CSQueue
@InterfaceAudience.Private public boolean getIntraQueuePreemptionDisabledInHierarchy()
CSQueue
getIntraQueuePreemptionDisabledInHierarchy
in interface CSQueue
@InterfaceAudience.Private public QueueCapacities getQueueCapacities()
CSQueue
getQueueCapacities
in interface CSQueue
@InterfaceAudience.Private public ResourceUsage getQueueResourceUsage()
CSQueue
getQueueResourceUsage
in interface CSQueue
public QueueResourceQuotas getQueueResourceQuotas()
CSQueue
getQueueResourceQuotas
in interface CSQueue
public ReentrantReadWriteLock.ReadLock getReadLock()
CSQueue
getReadLock
in interface CSQueue
public ReentrantReadWriteLock.WriteLock getWriteLock()
CSQueue
getWriteLock
in interface CSQueue
public Set<String> getConfiguredNodeLabels()
getConfiguredNodeLabels
in interface CSQueue
public void incReservedResource(String partition, org.apache.hadoop.yarn.api.records.Resource reservedRes)
Queue
incReservedResource
in interface Queue
partition
- asked by applicationreservedRes
- reserved resource askedpublic void decReservedResource(String partition, org.apache.hadoop.yarn.api.records.Resource reservedRes)
Queue
decReservedResource
in interface Queue
partition
- asked by applicationreservedRes
- reserved resource askedpublic void incPendingResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc)
Queue
incPendingResource
in interface Queue
nodeLabel
- asked by applicationresourceToInc
- new resource askedpublic void decPendingResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec)
CSQueue
decPendingResource
in interface CSQueue
decPendingResource
in interface Queue
nodeLabel
- asked by applicationresourceToDec
- new resource askedpublic void incUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToInc, SchedulerApplicationAttempt application)
CSQueue
incUsedResource
in interface CSQueue
nodeLabel
- node label.resourceToInc
- resource.application
- application.public void decUsedResource(String nodeLabel, org.apache.hadoop.yarn.api.records.Resource resourceToDec, SchedulerApplicationAttempt application)
CSQueue
decUsedResource
in interface CSQueue
nodeLabel
- node label.resourceToDec
- resource.application
- application.public org.apache.hadoop.yarn.api.records.Priority getDefaultApplicationPriority()
Queue
getDefaultApplicationPriority
in interface Queue
public Set<String> getNodeLabelsForQueue()
getNodeLabelsForQueue
in interface CSQueue
public org.apache.hadoop.yarn.api.records.Resource getTotalKillableResource(String partition)
public Iterator<RMContainer> getKillableContainers(String partition)
@VisibleForTesting public CSAssignment assignContainers(org.apache.hadoop.yarn.api.records.Resource clusterResource, FiCaSchedulerNode node, ResourceLimits resourceLimits, SchedulingMode schedulingMode)
assignContainers
in interface CSQueue
public boolean accept(org.apache.hadoop.yarn.api.records.Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,FiCaSchedulerNode> request)
public void validateSubmitApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId, String userName, String queue) throws org.apache.hadoop.security.AccessControlException
CSQueue
validateSubmitApplication
in interface CSQueue
applicationId
- Application IDuserName
- User Namequeue
- Queue Nameorg.apache.hadoop.security.AccessControlException
- if any acl violation is there.public void updateQueueState(org.apache.hadoop.yarn.api.records.QueueState queueState)
SchedulerQueue
updateQueueState
in interface SchedulerQueue<CSQueue>
queueState
- the queue statepublic void activateQueue() throws org.apache.hadoop.yarn.exceptions.YarnException
activateQueue
in interface SchedulerQueue<CSQueue>
org.apache.hadoop.yarn.exceptions.YarnException
- if its parent queue is not in RUNNING stateprotected void appFinished()
public org.apache.hadoop.yarn.api.records.Priority getPriority()
CSQueue
getPriority
in interface CSQueue
public UserWeights getUserWeights()
CSQueue
getUserWeights
in interface CSQueue
public void recoverDrainingState()
public String getMultiNodeSortingPolicyName()
CSQueue
getMultiNodeSortingPolicyName
in interface CSQueue
public void setMultiNodeSortingPolicyName(String policyName)
public long getMaximumApplicationLifetime()
CSQueue
getMaximumApplicationLifetime
in interface CSQueue
public long getDefaultApplicationLifetime()
CSQueue
getDefaultApplicationLifetime
in interface CSQueue
public boolean getDefaultAppLifetimeWasSpecifiedInConfig()
CSQueue
getDefaultAppLifetimeWasSpecifiedInConfig
in interface CSQueue
public void setMaxParallelApps(int maxParallelApps)
public int getMaxParallelApps()
CSQueue
getMaxParallelApps
in interface CSQueue
protected void updateAbsoluteCapacities()
public boolean isDynamicQueue()
CSQueue
isDynamicQueue
in interface CSQueue
public void setDynamicQueue(boolean dynamicQueue)
protected String getCapacityOrWeightString()
public boolean isEligibleForAutoDeletion()
public boolean isInactiveDynamicQueue()
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.