Parsing NameNode metrics data using Python urllib2 & json modules.

[#1]

- In this article we will see that in general how ambari makes use of the Python "urllib2" and "json" (simplejson) module to query and parse the API data.

- We will need to install the "simplejson" python module on our system.

- Now we can simply write following kind of python script "/tmp/NameNodeMetricsDetails.py"

import time
import json
import urllib2
import base64
import pprint 

# Ambari Admin credentials.
USERNAME = "admin"
PASSWORD = "admin"

# NameNode metrics.
query = "http://kjss1:8080/api/v1/clusters/JoyCluster/services/HDFS/components/NAMENODE"
connection_timeout = 30.0

def get_json_beans(query, connection_timeout):
  	response = None
  	try:
		raw = "%s:%s" % (USERNAME, PASSWORD)
		auth = 'Basic %s' % base64.b64encode(raw).strip()
		request = urllib2.Request(query)
		request.add_unredirected_header('Authorization', auth)  		
		response = urllib2.urlopen(request, timeout=connection_timeout)
		data = response.read()
		data_dict = json.loads(data)
		return data_dict
	finally:
		if response is not None:
			try:
				response.close()
			except:
				pass

def print_dictionary(d, indent=0):
	for key, value in d.iteritems():
		print '\t' * indent + str(key),
		if isinstance(value, dict):
			print ''
			print_dictionary(value, indent+1)
		else:
			print ('\t' * (indent+1) + str(value))

json_data = get_json_beans(query, connection_timeout)
print '------------------ DFS metrics ------------------'
dfs_metrics = json_data['metrics']['dfs']
print_dictionary(dfs_metrics, 1)

print '------------------ JVM metrics ------------------'
jvm_metrics = json_data['metrics']['jvm']
print_dictionary(jvm_metrics, 1)

- Now we can simply execute it as following:

$ python NameNodeMetricsDetails.py 

------------------ DFS metrics ------------------
	namenode 
		UpgradeFinalized 			True
		Used 			4929060864
		LiveNodes 			{"kjss2.example.com:1019":{"infoAddr":"10.20.69.238:1022","infoSecureAddr":"10.20.69.238:0","xferaddr":"10.20.69.238:1019","lastContact":1,"usedSpace":317841408,"adminState":"In Service","nonDfsUsedSpace":24614272775,"capacity":83339825152,"numBlocks":51,"version":"2.7.3.2.5.0.0-1182","used":317841408,"remaining":58407710969,"blockScheduled":0,"blockPoolUsed":317841408,"blockPoolUsedPercent":0.38138,"volfails":0},"kjss5.example.com:1019":{"infoAddr":"10.20.69.241:1022","infoSecureAddr":"10.20.69.241:0","xferaddr":"10.20.69.241:1019","lastContact":2,"usedSpace":1091743744,"adminState":"In Service","nonDfsUsedSpace":21535850247,"capacity":83339825152,"numBlocks":101,"version":"2.7.3.2.5.0.0-1182","used":1091743744,"remaining":60712231161,"blockScheduled":0,"blockPoolUsed":1091743744,"blockPoolUsedPercent":1.3099905,"volfails":0},"kjss1.example.com:1019":{"infoAddr":"10.20.69.237:1022","infoSecureAddr":"10.20.69.237:0","xferaddr":"10.20.69.237:1019","lastContact":2,"usedSpace":1185234944,"adminState":"In Service","nonDfsUsedSpace":25925525338,"capacity":83339825152,"numBlocks":71,"version":"2.7.3.2.5.0.0-1182","used":1185234944,"remaining":56229064870,"blockScheduled":0,"blockPoolUsed":1185234944,"blockPoolUsedPercent":1.4221712,"volfails":0},"kjss4.example.com:1019":{"infoAddr":"10.20.69.240:1022","infoSecureAddr":"10.20.69.240:0","xferaddr":"10.20.69.240:1019","lastContact":1,"usedSpace":1201344512,"adminState":"In Service","nonDfsUsedSpace":15253597959,"capacity":83339825152,"numBlocks":107,"version":"2.7.3.2.5.0.0-1182","used":1201344512,"remaining":66884882681,"blockScheduled":0,"blockPoolUsed":1201344512,"blockPoolUsedPercent":1.4415011,"volfails":0},"kjss3.example.com:1019":{"infoAddr":"10.20.69.239:1022","infoSecureAddr":"10.20.69.239:0","xferaddr":"10.20.69.239:1019","lastContact":2,"usedSpace":1132896256,"adminState":"In Service","nonDfsUsedSpace":14977347252,"capacity":83339825152,"numBlocks":99,"version":"2.7.3.2.5.0.0-1182","used":1132896256,"remaining":67229581644,"blockScheduled":0,"blockPoolUsed":1132896256,"blockPoolUsedPercent":1.3593696,"volfails":0}}
		TotalFiles 			354
		NameDirStatuses 			{"active":{"/hadoop/hdfs/namenode":"IMAGE_AND_EDITS"},"failed":{}}
		TotalBlocks 			130
		DecomNodes 			{}
		Free 			309463471325
		DeadNodes 			{}
		Version 			2.7.3.2.5.0.0-1182, r3aab7f2814e7017c194e648a1fec14719d5f3858
		Threads 			143
		PercentRemaining 			74.26545
		Safemode 			
		PercentUsed 			1.1828824
		CorruptFiles 			[]
		Total 			416699125760
		NonDfsUsedSpace 			102306593571
	FSNamesystem 
		CapacityTotalGB 			388.0
		MissingBlocks 			0
		CorruptBlocks 			0
		StaleDataNodes 			0
		CapacityUsedGB 			5.0
		PendingDataNodeMessageCount 			0
		CapacityRemainingGB 			288.0
		FilesTotal 			354
		TransactionsSinceLastLogRoll 			28
		PendingReplicationBlocks 			0
		PendingDeletionBlocks 			0
		UnderReplicatedBlocks 			24
		ScheduledReplicationBlocks 			0
		LastCheckpointTime 			1481854081000
		BlockCapacity 			2097152
		TotalFiles 			354
		CapacityTotal 			416699125760
		SnapshottableDirectories 			0
		BlocksTotal 			130
		LastWrittenTransactionId 			1882217
		ExpiredHeartbeats 			0
		ExcessBlocks 			0
		CapacityUsed 			4929060864
		TransactionsSinceLastCheckpoint 			11430
		MissingReplOneBlocks 			0
		Snapshots 			0
		CapacityRemaining 			309463471325
		PostponedMisreplicatedBlocks 			5
		MillisSinceLastLoadedEdits 			0
		TotalLoad 			40

------------------ JVM metrics ------------------
	logWarn 		239
	GcTimeMillisConcurrentMarkSweep 		232
	logError 		4
	threadsRunnable 		7
	memHeapCommittedM 		1004.0
	threadsWaiting 		11
	memMaxM 		1004.0
	threadsNew 		0
	threadsTimedWaiting 		125
	memHeapUsedM 		152.13956
	memNonHeapUsedM 		88.238945
	threadsTerminated 		0
	GcCountConcurrentMarkSweep 		2
	gcCount 		49
	logInfo 		36736
	threadsBlocked 		0
	logFatal 		0
	memNonHeapCommittedM 		90.27344
	gcTimeMillis 		1611

.