aboutsummaryrefslogtreecommitdiffstats
path: root/ticker.py
diff options
context:
space:
mode:
Diffstat (limited to 'ticker.py')
-rw-r--r--ticker.py25
1 files changed, 13 insertions, 12 deletions
diff --git a/ticker.py b/ticker.py
index 640b402..f7c1805 100644
--- a/ticker.py
+++ b/ticker.py
@@ -81,18 +81,19 @@ lines.append(line)
# This is not ideal. We should think of a way to record such failures.
# https://github.com/influxdata/influxdb/issues/4089
for gpu_node in [ 'vesta1', 'vesta2' ]:
- df, gpu_epoch, sucess = nvidia.read_gpu_stats(node=gpu_node, \
- gpu_type='tesla')
- for irow, [ index, row ] in enumerate(df.iterrows()):
- lines.append("gpu_temperature,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.gpu_temperature, gpu_epoch))
- if ~np.isnan(row.power_draw):
- lines.append("gpu_power_draw,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.power_draw, gpu_epoch))
- lines.append("gpu_utilization,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.gpu_utilization, gpu_epoch))
- lines.append("gpu_memory_utilization,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.memory_utilization, gpu_epoch))
+ df, gpu_epoch, success = nvidia.read_gpu_stats(node=gpu_node, \
+ gpu_type='tesla')
+ if success:
+ for irow, [ index, row ] in enumerate(df.iterrows()):
+ lines.append("gpu_temperature,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.gpu_temperature, gpu_epoch))
+ if ~np.isnan(row.power_draw):
+ lines.append("gpu_power_draw,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.power_draw, gpu_epoch))
+ lines.append("gpu_utilization,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.gpu_utilization, gpu_epoch))
+ lines.append("gpu_memory_utilization,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.memory_utilization, gpu_epoch))
# Join
data = "\n".join(lines)