Quản trị cụm Slurm HPC và các thực hành tốt nhất

1
# Add a new account/group
2
sacctmgr add account research_team Description="Research Team"
3

4
# Add a user to an account
5
sacctmgr add user john account=research_team
6

7
# Add user with multiple accounts
8
sacctmgr add user alice account=research_team,dev_team DefaultAccount=research_team
9

10
# View accounts
11
sacctmgr show account
12

13
# View users
14
sacctmgr show user

1
# Limit CPU minutes (prevents monopolizing cluster)
2
sacctmgr modify account research_team set GrpCPUMins=100000
3

4
# Limit memory (in MB)
5
sacctmgr modify account research_team set GrpMem=500000
6

7
# Limit concurrent jobs
8
sacctmgr modify account research_team set GrpJobs=50
9

10
# Limit concurrent running jobs
11
sacctmgr modify account research_team set GrpJobsRun=20
12

13
# Limit number of nodes
14
sacctmgr modify account research_team set GrpNodes=10

1
# Limit jobs in queue
2
sacctmgr modify user john set MaxJobs=10
3

4
# Limit running jobs
5
sacctmgr modify user john set MaxJobsRun=5
6

7
# Limit wall time (in minutes)
8
sacctmgr modify user john set MaxWall=1440  # 24 hours
9

10
# Limit CPUs per job
11
sacctmgr modify user john set MaxCPUs=32
12

13
# View user limits
14
sacctmgr show user john withassoc format=user,account,maxjobs,maxsubmit,maxwall

1
# Create QoS levels
2
sacctmgr add qos normal priority=100
3
sacctmgr add qos high priority=500 MaxWall=2-00:00:00 MaxJobs=5
4
sacctmgr add qos low priority=50
5

6
# Assign QoS to account
7
sacctmgr modify account research_team set qos=normal,high
8

9
# Users can specify QoS when submitting
10
sbatch --qos=high job_script.sh

1
# Set fair-share values (higher = more priority)
2
sacctmgr modify account research_team set fairshare=100
3
sacctmgr modify account dev_team set fairshare=50
4

5
# View fair-share tree
6
sshare -a
7

8
# View detailed fair-share info
9
sshare -A research_team --all

1
# View all nodes
2
sinfo
3

4
# Detailed node information
5
sinfo -Nel
6

7
# Show node states
8
sinfo -N -o "%N %T %C %m %e %f"
9

10
# View specific node details
11
scontrol show node worker-01

1
# Drain node (won't accept new jobs, allows running jobs to finish)
2
scontrol update NodeName=worker-01 State=drain Reason="Hardware upgrade"
3

4
# Force drain (terminate running jobs)
5
scontrol update NodeName=worker-01 State=drain Reason="Emergency maintenance"
6

7
# Check drain reason
8
sinfo -R

1
# Resume node
2
scontrol update NodeName=worker-01 State=resume
3

4
# Verify it's back
5
sinfo -n worker-01

1
# Mark node as down
2
scontrol update NodeName=worker-01 State=down Reason="Hardware failure"
3

4
# When fixed, resume
5
scontrol update NodeName=worker-01 State=resume

1
ansible-playbook -i inventories/hosts cluster_slurm.yml --limit worker-03

1
sudo systemctl restart slurmctld

1
sinfo
2
scontrol show node worker-03

1
# Controller logs
2
sudo tail -f /var/log/slurm/slurmctld.log
3

4
# Worker node logs (on compute nodes)
5
sudo tail -f /var/log/slurm/slurmd.log
6

7
# Database logs
8
sudo tail -f /var/log/slurm/slurmdbd.log
9

10
# Filter for errors
11
sudo grep "error" /var/log/slurm/*.log
12

13
# Filter for specific node
14
sudo grep "worker-01" /var/log/slurm/slurmctld.log
15

16
# Last 100 lines with context
17
sudo tail -100 /var/log/slurm/slurmctld.log

1
sinfo
2
# OUTPUT: worker-01    down   ...
3

4
scontrol show node worker-01
5
# Check "Reason" field

1
# 1. Check if slurmd is running
2
ssh worker-01 "sudo systemctl status slurmd"
3

4
# 2. Restart slurmd
5
ssh worker-01 "sudo systemctl restart slurmd"
6

7
# 3. Check network connectivity
8
ping worker-01
9

10
# 4. Check logs
11
ssh worker-01 "sudo tail -50 /var/log/slurm/slurmd.log"
12

13
# 5. Resume the node
14
scontrol update NodeName=worker-01 State=resume

1
squeue
2
# See jobs in PD (pending) state
3

4
# Check why job is pending
5
squeue --start -j JOB_ID
6

7
# View detailed job info
8
scontrol show job JOB_ID

1
# 1. Check available resources
2
sinfo -o "%P %a %l %D %N %C"
3

4
# 2. View job requirements
5
scontrol show job JOB_ID | grep -E "Partition|NumNodes|MinMemory"
6

7
# 3. Cancel job if needed
8
scancel JOB_ID
9

10
# 4. Modify pending job
11
scontrol update JobId=JOB_ID NumNodes=1

1
# Check job status
2
sacct -j JOB_ID
3

4
# View job output files
5
cat slurm-JOB_ID.out
6
cat slurm-JOB_ID.err

1
# Check slurmdbd status
2
sudo systemctl status slurmdbd
3

4
# Test database connection
5
sudo mysql -u slurm -p slurm_acct_db -e "SHOW TABLES;"
6

7
# Check slurmdbd logs
8
sudo tail -50 /var/log/slurm/slurmdbd.log

1
# 1. Restart slurmdbd
2
sudo systemctl restart slurmdbd
3

4
# 2. Verify database credentials in slurmdbd.conf
5
sudo cat /etc/slurm-llnl/slurmdbd.conf
6

7
# 3. Check database permissions
8
sudo mysql -e "SHOW GRANTS FOR 'slurm'@'localhost';"
9

10
# 4. Restart slurmctld to reconnect
11
sudo systemctl restart slurmctld

1
# On controller (rsyslog server)
2
sudo tail -f /var/log/syslog
3

4
# Filter by hostname
5
sudo grep "worker-01" /var/log/syslog
6

7
# Filter by service
8
sudo grep "slurmd" /var/log/syslog
9

10
# Check authentication logs
11
sudo tail -f /var/log/auth.log

1
# Generate key on your machine
2
ssh-keygen -t ed25519 -C "your_email@example.com"
3

4
# Copy to cluster
5
ssh-copy-id user@controller-node

1
# Verify munge is running
2
sudo systemctl status munge
3

4
# Test munge
5
munge -n | unmunge
6

7
# Generate new key (do this on controller, then distribute)
8
sudo /usr/sbin/create-munge-key
9

10
# Copy key to all nodes (Ansible does this automatically)
11
sudo scp /etc/munge/munge.key worker-01:/etc/munge/
12

13
# Restart munge on all nodes
14
sudo systemctl restart munge

1
# Allow Slurm ports only from cluster network
2
sudo ufw allow from 192.168.58.0/24 to any port 6817  # slurmctld
3
sudo ufw allow from 192.168.58.0/24 to any port 6818  # slurmd
4
sudo ufw allow from 192.168.58.0/24 to any port 6819  # slurmdbd
5

6
# Allow SSH from anywhere
7
sudo ufw allow 22/tcp
8

9
# Enable firewall
10
sudo ufw enable

1
# /etc/fstab on compute nodes
2
controller-01:/home /home nfs4 rw,soft,rsize=262144,wsize=262144,timeo=14,intr 0 0

1
/home/          # User home directories (SSD/NVMe)
2
  ├─ alice/
3
  ├─ bob/
4
  └─ charlie/
5

6
/mnt/data/      # Large datasets (HDD or object storage)
7
  ├─ shared/    # Common datasets
8
  ├─ projects/  # Project-specific data
9
  └─ scratch/   # Temporary data (auto-cleanup)
10

11
/opt/           # Shared software/modules
12
  ├─ anaconda/
13
  ├─ modules/
14
  └─ apps/

1
# Set user quotas
2
sudo setquota -u alice 50G 60G 0 0 /home
3
sudo setquota -u alice 500G 550G 0 0 /mnt/data
4

5
# Check quotas
6
quota -u alice
7

8
# View all quotas
9
sudo repquota -a

1
nextflow run nf-core/rnaseq -profile test,singularity --outdir test_rnaseq_profile_outdir

1
# Update cluster via Ansible
2
ansible-playbook -i inventories/hosts cluster_slurm.yml --tags update
3

4
# Update specific nodes
5
ansible-playbook -i inventories/hosts cluster_slurm.yml --limit worker-01,worker-02

1
# Backup Slurm configuration
2
sudo cp /etc/slurm-llnl/slurm.conf /backup/slurm.conf.$(date +%Y%m%d)
3

4
# Backup accounting database
5
sudo mysqldump -u slurm -p slurm_acct_db > slurm_acct_backup_$(date +%Y%m%d).sql
6

7
# Backup user data (use rsync for efficiency)
8
sudo rsync -av /home/ /backup/home/

1
# Check disk usage on all nodes
2
ansible all -i inventories/hosts -m shell -a "df -h"
3

4
# Check specific directory
5
ansible all -i inventories/hosts -m shell -a "du -sh /var/log/slurm"
6

7
# Find large files
8
find /home -type f -size +1G -exec ls -lh {} \;

1
# Use CR_CPU for CPU-bound jobs
2
SelectType=select/cons_tres
3
SelectTypeParameters=CR_CPU
4

5
# Or CR_Memory for memory-bound jobs
6
SelectTypeParameters=CR_Memory
7

8
# Or CR_Core for mixed workloads
9
SelectTypeParameters=CR_Core