aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVolker Hoffmann <volker@cheleb.net>2016-11-01 10:20:31 +0100
committerVolker Hoffmann <volker@cheleb.net>2016-11-01 10:20:31 +0100
commit74576ffb2806c5fc3afc3294968abf7f41023bc4 (patch)
tree9411f8ecdbe724094c113c56866f28d4ea31f72b
parent7fe62738902de67ef9b2698a5e3d55eef5441f46 (diff)
fix: do not choke if no nvidia-smi dumps are present
-rw-r--r--ticker.py25
1 files changed, 13 insertions, 12 deletions
diff --git a/ticker.py b/ticker.py
index 640b402..f7c1805 100644
--- a/ticker.py
+++ b/ticker.py
@@ -81,18 +81,19 @@ lines.append(line)
# This is not ideal. We should think of a way to record such failures.
# https://github.com/influxdata/influxdb/issues/4089
for gpu_node in [ 'vesta1', 'vesta2' ]:
- df, gpu_epoch, sucess = nvidia.read_gpu_stats(node=gpu_node, \
- gpu_type='tesla')
- for irow, [ index, row ] in enumerate(df.iterrows()):
- lines.append("gpu_temperature,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.gpu_temperature, gpu_epoch))
- if ~np.isnan(row.power_draw):
- lines.append("gpu_power_draw,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.power_draw, gpu_epoch))
- lines.append("gpu_utilization,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.gpu_utilization, gpu_epoch))
- lines.append("gpu_memory_utilization,node=%s,uuid=%s value=%.2f %i" % \
- (row.node, row.uuid, row.memory_utilization, gpu_epoch))
+ df, gpu_epoch, success = nvidia.read_gpu_stats(node=gpu_node, \
+ gpu_type='tesla')
+ if success:
+ for irow, [ index, row ] in enumerate(df.iterrows()):
+ lines.append("gpu_temperature,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.gpu_temperature, gpu_epoch))
+ if ~np.isnan(row.power_draw):
+ lines.append("gpu_power_draw,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.power_draw, gpu_epoch))
+ lines.append("gpu_utilization,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.gpu_utilization, gpu_epoch))
+ lines.append("gpu_memory_utilization,node=%s,uuid=%s value=%.2f %i" % \
+ (row.node, row.uuid, row.memory_utilization, gpu_epoch))
# Join
data = "\n".join(lines)