@InterfaceAudience.Private @Metrics(context="yarn") public class QueueMetrics extends Object implements org.apache.hadoop.metrics2.MetricsSource
Modifier and Type | Field and Description |
---|---|
protected org.apache.hadoop.conf.Configuration |
conf |
static String |
DEFAULT_PARTITION |
static String |
DEFAULT_PARTITION_JMX_STR |
static String |
METRIC_NAME_DELIMITER |
protected org.apache.hadoop.metrics2.MetricsSystem |
metricsSystem |
protected static org.apache.hadoop.metrics2.MetricsInfo |
P_RECORD_INFO |
protected static org.apache.hadoop.metrics2.MetricsInfo |
PARTITION_INFO |
protected static org.apache.hadoop.metrics2.MetricsInfo |
QUEUE_INFO |
protected String |
queueName |
protected org.apache.hadoop.metrics2.lib.MetricsRegistry |
registry |
protected Set<String> |
storedPartitionMetrics |
protected static org.apache.hadoop.metrics2.MetricsInfo |
USER_INFO |
protected Map<String,QueueMetrics> |
users |
Constructor and Description |
---|
QueueMetrics(org.apache.hadoop.metrics2.MetricsSystem ms,
String queueName,
Queue parent,
boolean enableUserMetrics,
org.apache.hadoop.conf.Configuration conf) |
Modifier and Type | Method and Description |
---|---|
void |
activateApp(String user) |
void |
addAppAttemptFirstContainerAllocationDelay(long latency) |
void |
allocateResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res,
boolean decrPending) |
void |
allocateResources(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource res)
Allocate Resource for container size change.
|
static void |
clearQueueMetrics()
Helper method to clear cache.
|
protected void |
createQueueMetricsForCustomResources() |
void |
deactivateApp(String user) |
void |
decrActiveUsers() |
void |
decrPendingResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
decrReserveResource(org.apache.hadoop.yarn.api.records.Resource res) |
void |
fillInValuesFromAvailableResources(org.apache.hadoop.yarn.api.records.Resource fromResource,
org.apache.hadoop.yarn.api.records.Resource targetResource)
Fills in Resource values from available metrics values of custom resources
to @code{targetResource}, only if the corresponding
value of @code{targetResource} is zero.
|
void |
finishApp(String user,
RMAppState rmAppFinalState,
boolean unmanagedAM) |
void |
finishAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId,
boolean isPending,
String user,
boolean unmanagedAM) |
static QueueMetrics |
forQueue(org.apache.hadoop.metrics2.MetricsSystem ms,
String queueName,
Queue parent,
boolean enableUserMetrics,
org.apache.hadoop.conf.Configuration conf) |
static QueueMetrics |
forQueue(String queueName,
Queue parent,
boolean enableUserMetrics,
org.apache.hadoop.conf.Configuration conf) |
int |
getActiveApps() |
int |
getActiveUsers() |
long |
getAggegatedReleasedContainers() |
long |
getAggregateAllocatedContainers() |
org.apache.hadoop.yarn.metrics.CustomResourceMetricValue |
getAggregatedPreemptedSecondsResources()
Handle this specially as this has a long value and it could be
truncated when casted into an int parameter of
Resource.newInstance (vCores).
|
long |
getAggregateMemoryMBPreempted() |
org.apache.hadoop.metrics2.lib.MutableCounterLong |
getAggregateMemoryMBSecondsPreempted() |
long |
getAggregateNodeLocalContainersAllocated() |
long |
getAggregateOffSwitchContainersAllocated() |
long |
getAggregatePreemptedContainers() |
long |
getAggregateRackLocalContainersAllocated() |
org.apache.hadoop.metrics2.lib.MutableCounterLong |
getAggregateVcoreSecondsPreempted() |
long |
getAggregateVcoresPreempted() |
int |
getAllocatedContainers() |
long |
getAllocatedMB() |
org.apache.hadoop.yarn.api.records.Resource |
getAllocatedResources() |
int |
getAllocatedVirtualCores() |
int |
getAppsCompleted() |
int |
getAppsFailed() |
int |
getAppsKilled() |
int |
getAppsPending() |
int |
getAppsRunning() |
int |
getAppsSubmitted() |
long |
getAvailableMB() |
org.apache.hadoop.yarn.api.records.Resource |
getAvailableResources() |
int |
getAvailableVirtualCores() |
void |
getMetrics(org.apache.hadoop.metrics2.MetricsCollector collector,
boolean all) |
org.apache.hadoop.metrics2.MetricsSystem |
getMetricsSystem() |
Queue |
getParentQueue() |
QueueMetrics |
getPartitionQueueMetrics(String partition)
Partition * Queue Metrics
Computes Metrics at Partition (Node Label) * Queue Level.
|
int |
getPendingContainers() |
long |
getPendingMB() |
org.apache.hadoop.yarn.api.records.Resource |
getPendingResources() |
int |
getPendingVirtualCores() |
static Map<String,QueueMetrics> |
getQueueMetrics()
Returns the metrics cache to help prevent re-registrations.
|
QueueMetricsForCustomResources |
getQueueMetricsForCustomResources() |
int |
getReservedContainers() |
long |
getReservedMB() |
org.apache.hadoop.yarn.api.records.Resource |
getReservedResources() |
int |
getReservedVirtualCores() |
int |
getUnmanagedAppsCompleted() |
int |
getUnmanagedAppsFailed() |
int |
getUnmanagedAppsPending() |
int |
getUnmanagedAppsRunning() |
int |
getUnmanagedAppsSubmitted() |
QueueMetrics |
getUserMetrics(String userName) |
void |
incrActiveUsers() |
void |
incrNodeTypeAggregations(String user,
NodeType type) |
void |
incrPendingResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res)
Increment pending resource metrics
|
void |
incrReserveResources(org.apache.hadoop.yarn.api.records.Resource res) |
void |
internalAllocateResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res,
boolean decrPending) |
protected void |
internalDecrPendingResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
internalIncrPendingResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
internalReleaseResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res) |
protected void |
internalReserveResources(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource res) |
protected void |
internalUnReserveResources(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource res) |
static boolean |
isConfigurationValidationSet(org.apache.hadoop.conf.Configuration conf)
Check whether we are in a configuration validation mode.
|
void |
moveAppFrom(AppSchedulingInfo app,
boolean unmanagedAM) |
void |
moveAppTo(AppSchedulingInfo app,
boolean unmanagedAM) |
void |
preemptContainer() |
protected void |
registerCustomResources() |
protected void |
registerPartitionMetricsCreation(String metricName) |
void |
releaseResources(String partition,
String user,
int containers,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
reserveResource(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
runAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String user,
boolean unmanagedAM) |
void |
setAvailableResources(org.apache.hadoop.yarn.api.records.Resource limit)
Set Available resources with support for resource vectors.
|
void |
setAvailableResourcesToQueue(org.apache.hadoop.yarn.api.records.Resource limit)
Set available resources.
|
void |
setAvailableResourcesToQueue(String partition,
org.apache.hadoop.yarn.api.records.Resource limit)
Set available resources.
|
void |
setAvailableResourcesToUser(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource limit)
Set available resources.
|
static void |
setConfigurationValidation(org.apache.hadoop.conf.Configuration conf,
boolean value)
Set configuration validation mode.
|
void |
setParent(QueueMetrics parent) |
void |
setParentQueue(Queue parentQueue) |
protected void |
setQueueMetricsForCustomResources(QueueMetricsForCustomResources metrics) |
protected static StringBuilder |
sourceName(String queueName) |
void |
submitApp(String user,
boolean unmanagedAM) |
void |
submitAppAttempt(String user,
boolean unmanagedAM) |
protected QueueMetrics |
tag(org.apache.hadoop.metrics2.MetricsInfo info,
String value) |
void |
unreserveResource(String partition,
String user,
org.apache.hadoop.yarn.api.records.Resource res) |
void |
updatePreemptedForCustomResources(org.apache.hadoop.yarn.api.records.Resource res) |
void |
updatePreemptedMemoryMBSeconds(long mbSeconds) |
void |
updatePreemptedResources(org.apache.hadoop.yarn.api.records.Resource res) |
void |
updatePreemptedSecondsForCustomResources(org.apache.hadoop.yarn.api.records.Resource res,
long seconds) |
void |
updatePreemptedVcoreSeconds(long vcoreSeconds) |
protected static final org.apache.hadoop.metrics2.MetricsInfo QUEUE_INFO
protected static final org.apache.hadoop.metrics2.MetricsInfo USER_INFO
protected static final org.apache.hadoop.metrics2.MetricsInfo PARTITION_INFO
protected final org.apache.hadoop.metrics2.lib.MetricsRegistry registry
protected final String queueName
protected final org.apache.hadoop.metrics2.MetricsSystem metricsSystem
protected final Map<String,QueueMetrics> users
protected final org.apache.hadoop.conf.Configuration conf
protected static final org.apache.hadoop.metrics2.MetricsInfo P_RECORD_INFO
public static final String DEFAULT_PARTITION
public static final String DEFAULT_PARTITION_JMX_STR
public static final String METRIC_NAME_DELIMITER
protected QueueMetrics tag(org.apache.hadoop.metrics2.MetricsInfo info, String value)
protected static StringBuilder sourceName(String queueName)
public static QueueMetrics forQueue(String queueName, Queue parent, boolean enableUserMetrics, org.apache.hadoop.conf.Configuration conf)
@InterfaceAudience.Private public static void clearQueueMetrics()
public static Map<String,QueueMetrics> getQueueMetrics()
QueueMetrics
map.public static QueueMetrics forQueue(org.apache.hadoop.metrics2.MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, org.apache.hadoop.conf.Configuration conf)
public QueueMetrics getUserMetrics(String userName)
public QueueMetrics getPartitionQueueMetrics(String partition)
partition
- Node Partitionpublic static boolean isConfigurationValidationSet(org.apache.hadoop.conf.Configuration conf)
conf
- the configuration to checkpublic static void setConfigurationValidation(org.apache.hadoop.conf.Configuration conf, boolean value)
conf
- the configuration to updatevalue
- the value for the validation modepublic void getMetrics(org.apache.hadoop.metrics2.MetricsCollector collector, boolean all)
getMetrics
in interface org.apache.hadoop.metrics2.MetricsSource
public void submitApp(String user, boolean unmanagedAM)
public void submitAppAttempt(String user, boolean unmanagedAM)
public void runAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId, String user, boolean unmanagedAM)
public void finishAppAttempt(org.apache.hadoop.yarn.api.records.ApplicationId appId, boolean isPending, String user, boolean unmanagedAM)
public void finishApp(String user, RMAppState rmAppFinalState, boolean unmanagedAM)
public void moveAppFrom(AppSchedulingInfo app, boolean unmanagedAM)
public void moveAppTo(AppSchedulingInfo app, boolean unmanagedAM)
public void setAvailableResourcesToQueue(String partition, org.apache.hadoop.yarn.api.records.Resource limit)
partition
- Node Partitionlimit
- resource limitpublic void setAvailableResources(org.apache.hadoop.yarn.api.records.Resource limit)
limit
- Resource.public void setAvailableResourcesToQueue(org.apache.hadoop.yarn.api.records.Resource limit)
limit
- resource limitpublic void setAvailableResourcesToUser(String partition, String user, org.apache.hadoop.yarn.api.records.Resource limit)
partition
- Node Partitionuser
- Name of the user.limit
- resource limitpublic void incrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
partition
- Node Partitionuser
- Name of the user.containers
- containers count.res
- the TOTAL delta of resources note this is different from the
other APIs which use per container resourcepublic void internalIncrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
protected void createQueueMetricsForCustomResources()
protected void registerCustomResources()
public void decrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
protected void internalDecrPendingResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
public void allocateResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res, boolean decrPending)
public void internalAllocateResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res, boolean decrPending)
public void allocateResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
partition
- Node Partitionuser
- Name of the userres
- Resource.public void releaseResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
public void internalReleaseResources(String partition, String user, int containers, org.apache.hadoop.yarn.api.records.Resource res)
public void preemptContainer()
public void updatePreemptedMemoryMBSeconds(long mbSeconds)
public void updatePreemptedVcoreSeconds(long vcoreSeconds)
public void updatePreemptedResources(org.apache.hadoop.yarn.api.records.Resource res)
public void updatePreemptedForCustomResources(org.apache.hadoop.yarn.api.records.Resource res)
public void updatePreemptedSecondsForCustomResources(org.apache.hadoop.yarn.api.records.Resource res, long seconds)
public void reserveResource(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
protected void internalReserveResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
public void incrReserveResources(org.apache.hadoop.yarn.api.records.Resource res)
public void unreserveResource(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
protected void internalUnReserveResources(String partition, String user, org.apache.hadoop.yarn.api.records.Resource res)
public void decrReserveResource(org.apache.hadoop.yarn.api.records.Resource res)
public void incrActiveUsers()
public void decrActiveUsers()
public void activateApp(String user)
public void deactivateApp(String user)
public void addAppAttemptFirstContainerAllocationDelay(long latency)
public int getAppsSubmitted()
public int getUnmanagedAppsSubmitted()
public int getAppsRunning()
public int getUnmanagedAppsRunning()
public int getAppsPending()
public int getUnmanagedAppsPending()
public int getAppsCompleted()
public int getUnmanagedAppsCompleted()
public int getAppsKilled()
public int getAppsFailed()
public int getUnmanagedAppsFailed()
public org.apache.hadoop.yarn.api.records.Resource getAllocatedResources()
public org.apache.hadoop.yarn.api.records.Resource getAvailableResources()
public org.apache.hadoop.yarn.api.records.Resource getPendingResources()
public org.apache.hadoop.yarn.api.records.Resource getReservedResources()
@VisibleForTesting public org.apache.hadoop.yarn.metrics.CustomResourceMetricValue getAggregatedPreemptedSecondsResources()
@VisibleForTesting public org.apache.hadoop.metrics2.lib.MutableCounterLong getAggregateMemoryMBSecondsPreempted()
@VisibleForTesting public org.apache.hadoop.metrics2.lib.MutableCounterLong getAggregateVcoreSecondsPreempted()
@VisibleForTesting public long getAggregateMemoryMBPreempted()
@VisibleForTesting public long getAggregateVcoresPreempted()
public long getAllocatedMB()
public int getAllocatedVirtualCores()
public int getAllocatedContainers()
public long getAvailableMB()
public int getAvailableVirtualCores()
public long getPendingMB()
public int getPendingVirtualCores()
public int getPendingContainers()
public long getReservedMB()
public int getReservedVirtualCores()
public int getReservedContainers()
public int getActiveUsers()
public int getActiveApps()
public org.apache.hadoop.metrics2.MetricsSystem getMetricsSystem()
public long getAggregateAllocatedContainers()
public long getAggregateNodeLocalContainersAllocated()
public long getAggregateRackLocalContainersAllocated()
public long getAggregateOffSwitchContainersAllocated()
public long getAggegatedReleasedContainers()
public long getAggregatePreemptedContainers()
public void fillInValuesFromAvailableResources(org.apache.hadoop.yarn.api.records.Resource fromResource, org.apache.hadoop.yarn.api.records.Resource targetResource)
fromResource
- The resource to compare available resource values with.targetResource
- The resource to save the values into.@VisibleForTesting public QueueMetricsForCustomResources getQueueMetricsForCustomResources()
protected void setQueueMetricsForCustomResources(QueueMetricsForCustomResources metrics)
public void setParent(QueueMetrics parent)
public Queue getParentQueue()
protected void registerPartitionMetricsCreation(String metricName)
public void setParentQueue(Queue parentQueue)
Copyright © 2008–2024 Apache Software Foundation. All rights reserved.