Slurm

scontrol update nodename=nodeXX state=resume

scontrol update jobid=jobid jobname=newname

scontrol update jobid=jobid partition=exclusive

scontrol show partition (partition_name)

scontrol show node (node_name)

scontrol show job jobid

scontrol hold/release/requeue jobid/joblist # use this command when jobs are waiting in queue

scontrol requeuehold/release jobid # use this command when jobs are running

sacctmgr list assoc/account (user=username)

sacct -u username -S 2022-07-01 -E now --field=jobid,partition,jobname,user,nnodes,nodelist,start,end,elapsed,alloccpus,state

$ systemctl restart slurmctld.service

$ systemctl restart slurmdbd.service

$ systemctl restart slurmd.service

$ systemctl restart slurmd.service

$ systemctl stop slurmdbd.service

$ systemctl stop slurmctld.service

$ systemctl stop slurmd.service

$ systemctl stop slurmd.service

$ mysqldump -p --databases slurm_acct_db > backup.sql (path: /root)

$ mkdir old_slurm

$ cp -r /etc/slurm/* /root/old_slurm

# conda deactivate, clean environment

# echo $LD_LIBRARY_PATH [*]

# unset LD_LIBRARY_PATH [*]

# export PATH=/usr/lib64/qt-3.3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/bin:/root/bin [*]

# yum --disablerepo='*' localinstall libaec-1.0.4-1.el7.x86_64.rpm libaec-devel-1.0.4-1.el7.x86_64.rpm hdf5-devel-1.8.12-12.el7.x86_64.rpm hdf5-1.8.12-12.el7.x86_64.rpm 

$ rpmbuild -ta slurm-*.tar.bz2

$ rpm --upgrade *.rpm # for downgrade, use "rpm -Uvh --oldpackage *.rpm"

$ yum --disablerepo='*' list slurm

# "[*]" maybe necessary

$ vim /etc/slurm/slurm.conf

#TaskPlugin=task/affinity

TaskPlugin=task/cgroup

PrologFlags=contain

$ vim /etc/pam.d/sshd

account    sufficient    pam_listfile.so item=user sense=allow onerr=fail file=/etc/ssh/slurm_allowed_users

-account    required      pam_slurm_adopt.so

$ vim /etc/pam.d/password-auth

#account    sufficient    pam_localuser.so

#-session   optional      pam_systemd.so

$ scp /etc/slurm/slurm.conf nodeXX:/etc/slurm/

$ scp /etc/pam.d/sshd nodeXX:/etc/pam.d/

$ scp /etc/pam.d/password-auth nodeXX:/etc/pam.d/password-auth

$ systemctl restart sshd.service

Failed to start MUNGE authentication service.

$ systemctl stop ntpd.service

$ ntpd -gq

$ systemctl start ntpd.service

# check if /var/run/munge is missing

$ mkdir /var/run/munge

$ chown -R munge:munge /var/run/munge/

$ systemctl restart munge.service

$ systemctl restart slurmd.service