ACCRE R9 Cluster Quick and Dirty Status
Report generated at Thu Jun 11 10:23:01 PM CDT 2026
Problem Nodes
HOSTNAMES STATE TIMESTAMP REASON COMMENT
cn1325 drain* 2026-06-11T08:10:44 Alex - RT99681 - Dim Alex - RT99681 - Dimm B4 replace
cn1498 drain* 2026-06-11T08:13:05 Alex - RT99311 - Fla Alex - RT99311 - Flapping network interface
cn1604 drng 2026-06-11T08:17:24 Troy - RT99656 - Pro Troy - RT99656 - Prolog error, panfs, drain, reboot, green,
gpu0208 drain 2026-06-11T08:18:49 Thomas - RT99603 - v Thomas - RT99603 - v5000/panfs hung clearing stuck CG
Queue Summary (Batch)
GROUP USER ACTIVE_JOBS ACTIVE_CORES PENDING_JOBS PENDING_CORES
-----------------------------------------------------------------------------------------
accre_guests 0 0 1 100
haojz 0 0 1 100
-----------------------------------------------------------------------------------------
beam_lab 22 522 150 1050
marshazm 15 480 25 800
pany11 5 10 125 250
zhuj29 2 32 0 0
-----------------------------------------------------------------------------------------
bias_group 1 10 0 0
biasds 1 10 0 0
-----------------------------------------------------------------------------------------
booth_lab 2 7 0 0
chenh55 1 4 0 0
comptoab 1 3 0 0
-----------------------------------------------------------------------------------------
castiglione_lab 0 0 1 8
jankeak 0 0 1 8
-----------------------------------------------------------------------------------------
cds_group 1 13 0 0
shress6 1 13 0 0
-----------------------------------------------------------------------------------------
cgg 0 0 1 64
liy110 0 0 1 64
-----------------------------------------------------------------------------------------
cms 371 4577 538 1306
cmslocal 224 2415 261 645
cmspilot 147 2162 277 661
-----------------------------------------------------------------------------------------
coxlab 2 15 0 0
blostf1 1 2 0 0
mille131 1 13 0 0
-----------------------------------------------------------------------------------------
cqs_si 0 0 4 8
chenarsw 0 0 4 8
-----------------------------------------------------------------------------------------
das_lab 1 1 0 0
shiltmh1 1 1 0 0
-----------------------------------------------------------------------------------------
davis_lab 0 0 1 16
bluejor 0 0 1 16
-----------------------------------------------------------------------------------------
econ_faculty 7 7 0 0
okeefm1 7 7 0 0
-----------------------------------------------------------------------------------------
escudero_lab 1 100 0 0
seifis1 1 100 0 0
-----------------------------------------------------------------------------------------
feng_lab 1 4 153 612
jiangl1 1 4 152 608
kilics1 0 0 1 4
-----------------------------------------------------------------------------------------
g_benntor_lab 1 8 26 185
delarr1 0 0 22 132
mccorcl1 1 8 4 53
-----------------------------------------------------------------------------------------
g_cohen_lab 0 0 2 2
wangg13 0 0 2 2
-----------------------------------------------------------------------------------------
g_gamazon_lab 1 4 0 0
salerl1 1 4 0 0
-----------------------------------------------------------------------------------------
goldring_group 1 5 200 1000
mcgrawke 1 5 200 1000
-----------------------------------------------------------------------------------------
gravity 10 40 60 210
ligo 10 40 60 210
-----------------------------------------------------------------------------------------
h_biostat_kang 0 0 400 400
yanb1 0 0 400 400
-----------------------------------------------------------------------------------------
h_biostat_student 149 212 7650 7650
liuk20 14 14 7650 7650
namy1 133 133 0 0
yangc16 1 1 0 0
yih4 1 64 0 0
-----------------------------------------------------------------------------------------
h_cqs 297 476 152 516
stolzej 2 32 0 0
yanb1 280 280 0 0
yangj24 15 164 152 516
-----------------------------------------------------------------------------------------
h_lu_lab 0 0 8 96
gaow9 0 0 8 96
-----------------------------------------------------------------------------------------
isde-rer 2 16 3 24
champaca 2 16 2 16
maybergm 0 0 1 8
-----------------------------------------------------------------------------------------
jswhep 1 8 0 0
yangt12 1 8 0 0
-----------------------------------------------------------------------------------------
kaczkurkin_lab 1 4 0 0
abbasia 1 4 0 0
-----------------------------------------------------------------------------------------
kojetin_lab 1 1 0 0
arifovl 1 1 0 0
-----------------------------------------------------------------------------------------
l3_aboud_lab 0 0 4 32
hongm1 0 0 4 32
-----------------------------------------------------------------------------------------
l3_jasper_lab 2 8 0 0
hattleee 1 2 0 0
jaspee1 1 6 0 0
-----------------------------------------------------------------------------------------
l3_precision_nutrition_lab 1 1 0 0
baghem1 1 1 0 0
-----------------------------------------------------------------------------------------
l3_runnoe_group 6 24 0 0
kaldorme 6 24 0 0
-----------------------------------------------------------------------------------------
lea_lab 3 12 23 92
songm6 3 12 23 92
-----------------------------------------------------------------------------------------
maha 0 0 1 1
wardbm1 0 0 1 1
-----------------------------------------------------------------------------------------
maiziezhou_lab 1 5 0 0
tangk10 1 5 0 0
-----------------------------------------------------------------------------------------
moro_lab 1 2 0 0
moroa 1 2 0 0
-----------------------------------------------------------------------------------------
nasa_imqcam 14 448 0 0
abuhamm 14 448 0 0
-----------------------------------------------------------------------------------------
nbody 7 464 1 1
pater32 0 0 1 1
smitm77 7 464 0 0
-----------------------------------------------------------------------------------------
ng_lab 1 8 0 0
kimj119 1 8 0 0
-----------------------------------------------------------------------------------------
nordman_lab 1 16 0 0
lit17 1 16 0 0
-----------------------------------------------------------------------------------------
okeefe_proj1 1 7 0 0
cornep1 1 7 0 0
-----------------------------------------------------------------------------------------
p_csb_meiler 171 1686 5614 8227
agarwm5 11 176 41 656
moreljl 10 10 3874 3874
mothcw 0 0 1000 1000
tydingcw 150 1500 222 2220
yange8 0 0 477 477
-----------------------------------------------------------------------------------------
p_dsi 0 0 2 3
yangi1 0 0 2 3
-----------------------------------------------------------------------------------------
p_englot_group 0 0 1 24
redaa1 0 0 1 24
-----------------------------------------------------------------------------------------
p_gordon_lab 1 5 0 0
rubatdtb 1 5 0 0
-----------------------------------------------------------------------------------------
p_matheny_lab 43 215 0 0
koolajd1 43 215 0 0
-----------------------------------------------------------------------------------------
p_meiler 0 0 1 3
yange8 0 0 1 3
-----------------------------------------------------------------------------------------
rer 9 104 0 0
cantrekb 1 4 0 0
karomnj 6 56 0 0
paciarja 1 20 0 0
wonge7 1 24 0 0
-----------------------------------------------------------------------------------------
rke_group 40 40 0 0
maduren 40 40 0 0
-----------------------------------------------------------------------------------------
rokaslab 6 86 12633 77858
copea1 1 1 0 0
danist 1 13 0 0
davidkt 3 24 6829 54632
riedlio 1 48 0 0
sautet1 0 0 5804 23226
-----------------------------------------------------------------------------------------
rubinov_lab 1 4 1 4
abbasia 1 4 0 0
rubinom 0 0 1 4
-----------------------------------------------------------------------------------------
sbcs 2 25 3 26
guoz18 0 0 2 6
lyul1 1 20 1 20
xus15 1 5 0 0
-----------------------------------------------------------------------------------------
stein_lab 1 8 0 0
karakg1 1 8 0 0
-----------------------------------------------------------------------------------------
taylor_group 1 3 0 0
petrop3 1 3 0 0
-----------------------------------------------------------------------------------------
tk_lab 2 80 0 0
yoonh15 2 80 0 0
-----------------------------------------------------------------------------------------
tplab 2 16 0 0
walkerdg 2 16 0 0
-----------------------------------------------------------------------------------------
vgi 41 264 1231 6879
gaow9 35 210 269 1614
parkj71 5 50 962 5265
salerl1 1 4 0 0
-----------------------------------------------------------------------------------------
walker_lab 87 88 0 0
deanrt 1 2 0 0
mcdanjm2 85 85 0 0
walkeas2 1 1 0 0
-----------------------------------------------------------------------------------------
wankowicz_lab 53 53 22 175
beckermf 0 0 22 175
wankows 53 53 0 0
-----------------------------------------------------------------------------------------
wan_lab 2 140 0 0
hardenn 2 140 0 0
-----------------------------------------------------------------------------------------
williams_roberson_lab 1 1 0 0
yeohb1 1 1 0 0
-----------------------------------------------------------------------------------------
yang_lab_csb 12 216 5 90
zhengm9 12 216 5 90
-----------------------------------------------------------------------------------------
Totals: 1386 10059 28892 106662
Queue Summary (Batch GPU)
GROUP USER ACTIVE_JOBS ACTIVE_GPUS PENDING_JOBS PENDING_GPUS
-----------------------------------------------------------------------------------------
accre_gpu_acc 1 4 0 0
koirap1 1 4 0 0
-----------------------------------------------------------------------------------------
accre_guests_acc 5 5 0 0
liy110 2 2 0 0
wanj119 1 1 0 0
yooj11 2 2 0 0
-----------------------------------------------------------------------------------------
beam_lab_acc 1 1 0 0
pany11 1 1 0 0
-----------------------------------------------------------------------------------------
bennartz_lab_acc 1 1 0 0
guptaak4 1 1 0 0
-----------------------------------------------------------------------------------------
csb_gpu_acc 6 9 755 761
agarwm5 0 0 753 753
arifovl 2 2 0 0
dongj11 1 1 0 0
huntek1 1 1 0 0
karadim 1 4 2 8
lybrantp 1 1 0 0
-----------------------------------------------------------------------------------------
es3890_acc 1 1 0 0
sadridm 1 1 0 0
-----------------------------------------------------------------------------------------
h_oguz_lab_acc 1 1 0 0
wanj119 1 1 0 0
-----------------------------------------------------------------------------------------
h_vmac_acc 0 0 1 1
janveva 0 0 1 1
-----------------------------------------------------------------------------------------
maple_lab_acc 0 0 1 1
lif12 0 0 1 1
-----------------------------------------------------------------------------------------
mltf_acc 2 13 0 0
sheny19 1 12 0 0
wuy55 1 1 0 0
-----------------------------------------------------------------------------------------
nbody_acc 1 1 0 0
bustam1 1 1 0 0
-----------------------------------------------------------------------------------------
oliver_lab_acc 32 32 0 0
gonzc11 32 32 0 0
-----------------------------------------------------------------------------------------
p_dsi_acc 0 0 4 4
yangi1 0 0 4 4
-----------------------------------------------------------------------------------------
p_meiler_acc 3 3 0 0
labeilro 2 2 0 0
moreljl 1 1 0 0
-----------------------------------------------------------------------------------------
taylor_group_acc 2 3 0 0
laaln1 1 2 0 0
schultls 1 1 0 0
-----------------------------------------------------------------------------------------
Totals: 56 74 761 767
Queue Summary (interactive)
GROUP USER ACTIVE_JOBS ACTIVE_CORES PENDING_JOBS PENDING_CORES
-----------------------------------------------------------------------------------------
booth_lab_int 2 3 0 0
mathura 1 2 0 0
wanj129 1 1 0 0
-----------------------------------------------------------------------------------------
l3_precision_nutrition_lab_int 1 128 0 0
baghem1 1 128 0 0
-----------------------------------------------------------------------------------------
maiziezhou_lab_int 12 60 91 455
tangk10 12 60 91 455
-----------------------------------------------------------------------------------------
nbody_int 1 36 1 25
carrilm 0 0 1 25
smitm77 1 36 0 0
-----------------------------------------------------------------------------------------
rubinov_lab_int 3 34 0 0
mohamb2 1 16 0 0
rubinom 1 8 0 0
sardarn 1 10 0 0
-----------------------------------------------------------------------------------------
yang_lab_int 1 8 0 0
shaoq1 1 8 0 0
-----------------------------------------------------------------------------------------
Totals: 20 269 92 480
Queue Summary (interactive_gpu)
GROUP USER ACTIVE_JOBS ACTIVE_GPUS PENDING_JOBS PENDING_GPUS
-----------------------------------------------------------------------------------------
accre_dgx_iacc 1 1 0 0
donovcl1 1 1 0 0
-----------------------------------------------------------------------------------------
dsi_dgx_iacc 4 7 0 0
chattec 1 1 0 0
samkn 1 1 0 0
schultls 1 1 0 0
wut18 1 4 0 0
-----------------------------------------------------------------------------------------
p_matheny_lab_iacc 1 1 0 0
koolajd1 1 1 0 0
-----------------------------------------------------------------------------------------
Totals: 6 9 0 0
Partition Summary
PARTITION AVAIL TIMELIMIT NODES STATE NODELIST
interactive up 14-00:00:0 1 drain* cn1325
interactive up 14-00:00:0 11 mix cn[1287,1301-1302,1322-1324,1328-1329,1812-1814]
interactive up 14-00:00:0 2 alloc cn[1326,1800]
interactive up 14-00:00:0 16 idle cn[1330,1707,1801-1811,1815-1817]
batch* up 14-00:00:0 4 mix- cn[1538,1550,1569,1580]
batch* up 14-00:00:0 1 drain* cn1498
batch* up 14-00:00:0 1 drng cn1604
batch* up 14-00:00:0 211 mix cn[1202-1207,1209-1213,1215,1217,1219,1229-1231,1235-1236,1239-1241,1257-1261,1264-1266,1268,1270,1274-1275,1277-1284,1288,1291,1294-1299,1303,1306-1307,1309-1314,1316-1318,1320-1321,1327,1332-1336,1338-1340,1342,1346,1348-1349,1352,1357-1360,1362-1364,1367,1369,1371,1377-1385,1387-1390,1392-1394,1396,1398-1411,1422,1427,1431,1435-1436,1438-1439,1443,1445,1447-1449,1453-1454,1456,1460-1462,1464,1466-1467,1469-1470,1473,1475,1478-1479,1496,1499,1501,1508,1510-1513,1517,1523,1528-1530,1532-1533,1537,1543-1544,1548,1552,1559,1562-1563,1565,1568,1570-1571,1581-1583,1585,1587,1589,1593-1594,1596,1598,1601-1603,1605-1607,1612,1616-1621,1625,1629-1630,1632-1633,1700-1705,1708-1710,2000]
batch* up 14-00:00:0 170 alloc cn[1208,1216,1218,1220-1224,1226-1228,1232-1234,1237-1238,1242,1262,1267,1269,1271-1273,1285-1286,1289-1290,1292-1293,1304-1305,1308,1315,1331,1337,1341,1343-1345,1347,1350-1351,1353-1354,1361,1365-1366,1368,1372-1376,1391,1395,1397,1414-1421,1423-1426,1430,1432,1434,1437,1441-1442,1446,1450,1452,1455,1457-1458,1463,1468,1471-1472,1474,1476-1477,1480-1495,1497,1500,1502-1507,1509,1514-1516,1518-1520,1522,1524-1525,1527,1531,1534-1536,1545-1547,1549,1551,1553-1558,1561,1564,1566-1567,1573-1579,1584,1586,1588,1592,1595,1597,1599-1600,1608-1610,1613-1615,1622-1624,1631,1706,1711-1713]
batch* up 14-00:00:0 4 idle cn[1626-1628,1714]
batch_gpu up 14-00:00:0 1 drain gpu0208
batch_gpu up 14-00:00:0 18 mix gpu[0059,0062-0066,0070-0072,0076-0079,0081-0082,0302],gracehopper01,hgx03
batch_gpu up 14-00:00:0 5 alloc gpu[0067-0069,0075,0080]
batch_gpu up 14-00:00:0 16 idle gpu[0073-0074,0084-0085,0300-0301,0303-0310],gracehopper02,hgx02
interactive_gpu up 14-00:00:0 3 mix dgx[01,03],gpu0207
interactive_gpu up 14-00:00:0 4 idle dgx04,gpu[0058,0060-0061]
sam up 2-02:00:00 1 alloc cms-sam-01
sam up 2-02:00:00 1 idle cms-sam-02
reserved inact infinite 0 n/a