Quick help commands: HBase, HiveServer2, Oozie, Spark, Yarn, Zookeeper, some general system commands

[#1]

1 : Check if hbase-master is running

sudo /etc/init.d/hbase-master status

if not, then start it sudo /etc/init.d/hbase-master start

2 : Check if hbase-regionserver is running

sudo /etc/init.d/hbase-regionserver status

if not, then start it sudo /etc/init.d/hbase-regionserver start

3 : Check if zookeeper-server is running

sudo /etc/init.d/zookeeper-server status

if not, then start it sudo /etc/init.d/zookeeper-server start

4: Grepping for open port

netstat -apn |grep <port to look for>

5: Process memory usage

ps -eo pmem,pcpu,vsize,pid,cmd | sort -k 1 -nr | head -5

or

ps -A --sort -rss -o comm,pmem | head -n 11

6: Free memory on system: CentOS

free -g

7: Yarn application log checking

yarn logs -applicationId <application ID>

dig deeper via suing the container id along with application id

yarn logs -applicationId <application ID> -containerId <container id>

8: starting zkCli cli connection to server

cd /grid/0/hdp/current/zookeeper-client/bin

And

./zkCli.sh -server <zookeeper server fqdn>:2181

9: starting name node from cli

su -l hdfs -c "/usr/hdp/current/hadoop-hdfs-namenode/../hadoop/sbin/hadoop-daemon.sh start namenode"

10: Spark server - starting it when missing hdfs directory

hdfs dfs -mkdir /spark-history

hdfs dfs -chown -R spark:hadoop /spark-history

hdfs dfs -chmod -R 777 /spark-history

su - spark -c "/usr/hdp/current/spark-historyserver/sbin/start-history-server.sh"

11: Start History server::: (mapreduce) from cli

su -l mapred -c "/usr/hdp/current/hadoop-mapreduce-historyserver/sbin/mr-jobhistory-daemon.sh start historyserver"

12: Starting App timeline server (Yarn) from cli

su - yarn

/grid/0/hdp/2.5.3.0-37/hadoop-yarn/sbin/yarn-daemon.sh start timelineserver

13: start and stop oozie server from CLI (on the machine where Ozzie server is installed)

su oozie

/usr/hdp/current/oozie-server/bin/oozied.sh start

/usr/hdp/current/oozie-server/bin/oozied.sh stop

14: Hbase mater (Hbase) from cli

Go to cluster node where Hbase master is installed, and then-

su -l hbase -c "/usr/hdp/current/hbase-master/bin/hbase-daemon.sh start master; sleep 25"

15: Hive2 server (Hive) from cli

Go to cluster node where HiveServer2 is installed, then-

su hive nohup /usr/hdp/current/hive-server2/bin/hiveserver2 -hiveconf hive.metastore.uris=/tmp/hiveserver2HD.out 2 /tmp/hiveserver2HD.log

Or

su - hive -l -c 'HIVE_CONF_DIR=/etc/hive/conf /usr/hdp/current/hive-server2/bin/hiveserver2 -hiveconf hive.metastore.uris="" -hiveconf hive.log.dir=/var/log/hive -hiveconf hive.log.file=hiveserver2.log 1>/var/log/hive/hiveserver2.log 2>/var/log/hive/hiveserver2.log &'

Or

sudo su - -c "export HIVE_CONF_DIR=/tmp/hiveConf;nohup /usr/hdp/current/hive-server2/bin/hiveserver2 -hiveconf hive.metastore.uris=' ' -hiveconf hive.log.file=hiveServer2.log -hiveconf hive.log.dir=/var/log/hive > /var/log/hive/hiveServer2.out 2>> /var/log/hive/hiveServer2.log &" hive

[root@stlrx2540m1-109 ~]# ps -ef | grep HiveSer

[root@stlrx2540m1-109 ~]# echo 28627 > /var/run/hive/hive-server.pid

[root@stlrx2540m1-109 ~]# echo 28627 > /var/run/hive/hive.pid

[root@stlrx2540m1-109 ~]# chmod 644 /var/run/hive/hive-server.pid

[root@stlrx2540m1-109 ~]# chmod 644 /var/run/hive/hive.pid

[root@stlrx2540m1-109 ~]# chown hive:hadoop /var/run/hive/hive-server.pid

[root@stlrx2540m1-109 ~]# chown hive:hadoop /var/run/hive/hive.pid