Member since 
    
	
		
		
		12-06-2016
	
	
	
	
	
	
	
	
	
	
	
	
	
	
			
      
                136
            
            
                Posts
            
        
                12
            
            
                Kudos Received
            
        
                1
            
            
                Solution
            
        My Accepted Solutions
| Title | Views | Posted | 
|---|---|---|
| 3539 | 01-18-2018 12:56 PM | 
			
    
	
		
		
		03-02-2018
	
		
		02:03 PM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Also,  may be you have more  information about   Hbase & Elasticsearch  & ) 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		03-01-2018
	
		
		03:00 PM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
	
		2 Kudos
		
	
				
		
	
		
					
							 It is posible Send Ambari Metric to Elasticsearch  ?   
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
		
			
				
						
							Labels:
						
						
		
			
	
					
			
		
	
	
	
	
				
		
	
	
- Labels:
- 
						
							
		
			Apache Ambari
			
    
	
		
		
		01-18-2018
	
		
		12:56 PM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Working after add in test_alert_disk_space.py  next lines :  custom_mount_point = ['/usr','/hadoop','/var','/var/lib','/home','/boot']
# for l in mountPointsList:
  for l in custom_mount_point:
        print "---------- l : " + l
        if  l:
 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-17-2018
	
		
		08:04 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 spark2-error.txt   <<<  Errors detail  
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-17-2018
	
		
		07:59 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 HDP-2.6.3.0  /usr/hdp/current/spark2-client/bin/spark-submit dpi_test.py --queue load --driver-memory 10g --num-executors 6 --executor-memory 30G  dpi_test.py :
from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql import HiveContext
from pyspark.sql.functions import udf, col
from pyspark.sql.types import BooleanType
import re
import time
warehouse_location = abspath('spark-warehouse')
spark = SparkSession \
    .builder \
    .appName("Python Spark SQL Hive integration example") \
    .config("spark.sql.warehouse.dir", warehouse_location) \
    .enableHiveSupport() \
    .getOrCreate()
cont=udf(lambda x,y: bool(re.match('^((.)*([^0-9A-Za-z])+)*'+x+'(([^0-9A-Za-z])+(.)*)*$',y))&True, BooleanType())
spark.sql("select d.msisdn, d.end_time from other_sources.stg_dpi_other_day d where msisdn='380988526911'").groupby('msisdn').agg({'end_time':'min', 'end_time':'max','end_time':'count'}).show(n=50)
 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
		
			
				
						
							Labels:
						
						
		
			
	
					
			
		
	
	
	
	
				
		
	
	
- Labels:
- 
						
							
		
			Apache Spark
			
    
	
		
		
		01-17-2018
	
		
		06:43 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 We need monitoring not only :   STACK_HOME_DIR = "/usr/hdp"    or   STACK_HOME_LEGACY_DIR = "/usr/lib"  but and another directories on all nodes cluster.  In Ambari I see same errors :     
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-16-2018
	
		
		07:17 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 How to limit mount point or resolve error : OSError: [Errno 2] No such file or directory: 'net_prio'  ?  Ambari Version 2.5.0.3 (HDP-2.6.3.0)  [root@serv03 host_scripts]# python test_alert_disk_space.py
mountPoints = ,/sys,/proc,/dev,/sys/kernel/security,/dev/shm,/dev/pts,/run,/sys/fs/cgroup,/sys/fs/cgroup/systemd,/sys/fs/pstore,/sys/fs/cgroup/pids,/sys/fs/cgroup/net_cls,net_prio,/sys/fs/cgroup/perf_event,/sys/fs/cgroup/cpu,cpuacct,/sys/fs/cgroup/devices,/sys/fs/cgroup/hugetlb,/sys/fs/cgroup/blkio,/sys/fs/cgroup/memory,/sys/fs/cgroup/cpuset,/sys/fs/cgroup/freezer,/sys/kernel/config,/,/usr,/proc/sys/fs/binfmt_misc,/dev/mqueue,/sys/kernel/debug,/dev/hugepages,/hadoop,/var,/var/lib,/home,/boot,/proc/sys/fs/binfmt_misc,/run/user/1015,/run/user/1012,/run/user/1031,/run/user/0,/run/user/1029,/run/user/1010,/run/user/1037,/run/user/1013,/run/user/1018,/run/user/1040,/run/user/1022,/run/user/1039
['', '/sys', '/proc', '/dev', '/sys/kernel/security', '/dev/shm', '/dev/pts', '/run', '/sys/fs/cgroup', '/sys/fs/cgroup/systemd', '/sys/fs/pstore', '/sys/fs/cgroup/pids', '/sys/fs/cgroup/net_cls', 'net_prio', '/sys/fs/cgroup/perf_event', '/sys/fs/cgroup/cpu', 'cpuacct', '/sys/fs/cgroup/devices', '/sys/fs/cgroup/hugetlb', '/sys/fs/cgroup/blkio', '/sys/fs/cgroup/memory', '/sys/fs/cgroup/cpuset', '/sys/fs/cgroup/freezer', '/sys/kernel/config', '/', '/usr', '/proc/sys/fs/binfmt_misc', '/dev/mqueue', '/sys/kernel/debug', '/dev/hugepages', '/hadoop', '/var', '/var/lib', '/home', '/boot', '/proc/sys/fs/binfmt_misc', '/run/user/1015', '/run/user/1012', '/run/user/1031', '/run/user/0', '/run/user/1029', '/run/user/1010', '/run/user/1037', '/run/user/1013', '/run/user/1018', '/run/user/1040', '/run/user/1022', '/run/user/1039']
---------- l :
FINAL finalResultCode  CODE .....
---------- l : /sys
/sys
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /proc
/proc
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /dev
/dev
disk_usage.total
134933655552
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/kernel/security
/sys/kernel/security
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /dev/shm
/dev/shm
disk_usage.total
134944870400
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /dev/pts
/dev/pts
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /run
/run
disk_usage.total
134944870400
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/fs/cgroup
/sys/fs/cgroup
disk_usage.total
134944870400
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/fs/cgroup/systemd
/sys/fs/cgroup/systemd
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/fs/pstore
/sys/fs/pstore
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/fs/cgroup/pids
/sys/fs/cgroup/pids
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : /sys/fs/cgroup/net_cls
/sys/fs/cgroup/net_cls
disk_usage.total = 0
=>WARNING
FINAL finalResultCode  CODE .....WARNING
---------- l : net_prio
net_prio
Traceback (most recent call last):
  File "test_alert_disk_space.py", line 241, in <module>
    execute()
  File "/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py", line 89, in thunk
    return fn(*args, **kwargs)
  File "test_alert_disk_space.py", line 90, in execute
    disk_usage = _get_disk_usage(mountPath)
  File "/usr/lib/python2.6/site-packages/ambari_commons/os_family_impl.py", line 89, in thunk
    return fn(*args, **kwargs)
  File "test_alert_disk_space.py", line 179, in _get_disk_usage
    disk_stats = os.statvfs(path)
OSError: [Errno 2] No such file or directory: 'net_prio'
[root@serv03 host_scripts]#
 
 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
		
			
				
						
							Labels:
						
						
		
			
	
					
			
		
	
	
	
	
				
		
	
	
- Labels:
- 
						
							
		
			Apache Ambari
			
    
	
		
		
		01-16-2018
	
		
		07:14 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 ok!   Thank you! 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		 
        













