Tips to Improve HDFS Rebalancer

[#1]

dfs.datanode.max.xcievers / dfs.datanode.max.transfer.threads = 4096 (use 16k if running HBase) 


dfs.datanode.balance.max.concurrent.moves = 500 (can go to 1000 if needed) 


/* Each  data  node  has  a  limited  bandwidth  for  rebalancing.  The  default  value  for  the  bandwidth is 5MB/s.  In  the  worst  case,  each  data  transfer  has  a  limited  
bandwidth of 1MB/s. Default is dfs.datanode.balance.bandwidthPerSec = 5242880 */

dfs.datanode.balance.bandwidthPerSec = 104857600 /* 100 MB/s */ 

hdfs balancer -Dfs.defaultFS=hdfs://<NN_HOSTNAME>:8020 -Ddfs.balancer.movedWinWidth=5400000 -Ddfs.balancer.moverThreads=1000 -Ddfs.balancer.dispatcherThreads=200 -Ddfs.datanode.balance.max.concurrent.moves=5 -Ddfs.balance.bandwidthPerSec=100000000 -Ddfs.balancer.max-size-to-move=10737418240 -threshold 5