CDH Cluster has been online more than half year, all the machines and service is normal.
but recently , there is one host agent always under bad healthy, and Cloudera Manager will show the red color on this host.
agent cpu usage 90%+
agent version: 6.3.1
agent pid:40892
run cmd:
ps -eLo pid,lwp,pcpu |grep 40892
lwp 42208 cpu usage high
check this thread: 42208
Thread 3 (Thread 0x7f73cdffb700 (LWP 42208)):
#0 0x00007f745386b4c2 in __memcpy_ssse3 () from /lib64/libc.so.6
#1 0x00007f74544a0268 in string_concat () from /lib64/libpython2.7.so.1.0
#2 0x00007f74544a2015 in PyString_Concat () from /lib64/libpython2.7.so.1.0
#3 0x00007f7454450c71 in string_concatenate () from /lib64/libpython2.7.so.1.0
#4 0x00007f74544f3d39 in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#5 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#6 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#7 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#8 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#9 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#10 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#11 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#12 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#13 0x00007f74544f3b4c in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#14 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#15 0x00007f745447ff88 in function_call () from /lib64/libpython2.7.so.1.0
#16 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#17 0x00007f745446a065 in instancemethod_call () from /lib64/libpython2.7.so.1.0
#18 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#19 0x00007f74544b2437 in slot_tp_call () from /lib64/libpython2.7.so.1.0
#20 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#21 0x00007f74544ef846 in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#22 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#23 0x00007f745447ff88 in function_call () from /lib64/libpython2.7.so.1.0
#24 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#25 0x00007f745446a065 in instancemethod_call () from /lib64/libpython2.7.so.1.0
#26 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#27 0x00007f74544b2437 in slot_tp_call () from /lib64/libpython2.7.so.1.0
#28 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#29 0x00007f74544ef846 in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#30 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#31 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#32 0x00007f745448007d in function_call () from /lib64/libpython2.7.so.1.0
#33 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#34 0x00007f745446a065 in instancemethod_call () from /lib64/libpython2.7.so.1.0
#35 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#36 0x00007f74544b2097 in slot_tp_init () from /lib64/libpython2.7.so.1.0
#37 0x00007f74544b0daf in type_call () from /lib64/libpython2.7.so.1.0
#38 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#39 0x00007f74544ef846 in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#40 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#41 0x00007f74544f3b4c in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#42 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#43 0x00007f745447ff88 in function_call () from /lib64/libpython2.7.so.1.0
#44 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#45 0x00007f745446a065 in instancemethod_call () from /lib64/libpython2.7.so.1.0
#46 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#47 0x00007f74544ef846 in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#48 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#49 0x00007f74544f3b4c in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#50 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#51 0x00007f74544f3b4c in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#52 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#53 0x00007f745448007d in function_call () from /lib64/libpython2.7.so.1.0
#54 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#55 0x00007f74544eed0d in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#56 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#57 0x00007f74544f3b4c in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#58 0x00007f74544f3ccd in PyEval_EvalFrameEx () from /lib64/libpython2.7.so.1.0
#59 0x00007f74544f664d in PyEval_EvalCodeEx () from /lib64/libpython2.7.so.1.0
#60 0x00007f745447ff88 in function_call () from /lib64/libpython2.7.so.1.0
#61 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#62 0x00007f745446a065 in instancemethod_call () from /lib64/libpython2.7.so.1.0
#63 0x00007f745445b073 in PyObject_Call () from /lib64/libpython2.7.so.1.0
#64 0x00007f74544ecf07 in PyEval_CallObjectWithKeywords () from /lib64/libpython2.7.so.1.0
#65 0x00007f7454524e42 in t_bootstrap () from /lib64/libpython2.7.so.1.0
#66 0x00007f74541faea5 in start_thread () from /lib64/libpthread.so.0
#67 0x00007f745381a96d in clone () from /lib64/libc.so.6
kill -SIGQUIT 40892
cat cloudera-scm-agent.log
Dumping all Thread Stacks ...
# Thread: Monitor-GenericMonitor(140135345661696)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: ImpalaDaemonQueryMonitoring(140135354054400)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/stoppable_thread.py", line 28, in run
self._fn(*self._args, **self._kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 849, in _check_for_queries
self._get_completed_query_profiles()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 900, in _get_completed_query_profiles
self._query_monitor.get_completed_queries(query_log_file)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 580, in get_completed_queries
completed_query_report_limit)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 455, in get_completed_queries
last_accessed_file_timestamp)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 256, in _get_completed_queries
file_filter=filters)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/event_streamer.py", line 114, in __init__
self.__filtered_file_list = self.__apply_file_filter()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/event_streamer.py", line 186, in __apply_file_filter
self.__file_filter(filter_context)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/common/chain.py", line 22, in __call__
succ = command(context)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 94, in __call__
self.__set_start_offset(f, evt1.get_datetime())
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/impalad/query_monitor.py", line 106, in __set_start_offset
event = self.__event_reader.find_nearest_event_datetime(self.__start_date_time)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/event_reader.py", line 102, in find_nearest_event_datetime
event = self.find_event_containing_pos(mid)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/event_reader.py", line 124, in find_event_containing_pos
self.__line_reader.goto_location(pos)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/file_line_reader.py", line 124, in goto_location
self.__read_data_till_prev_newline()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/clusterstats/log/streaming/file_line_reader.py", line 172, in __read_data_till_prev_newline
self.__file_handle.seek(-seek_len, 1)
# Thread: MonitorDaemon-Reporter(140135924496128)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 51, in run
self._fn(*self._args, **self._kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/daemon.py", line 167, in _report
self._report_for_monitors(monitors)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/daemon.py", line 194, in _report_for_monitors
role_update = self._safe_get_role_update(monitor)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/daemon.py", line 217, in _safe_get_role_update
return monitor.get_role_update()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/generic/__init__.py", line 204, in get_role_update
LOG)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/proc_metrics_utils.py", line 184, in add_with_descendants_proc_metrics
metrics = _get_process_and_descendant_metrics(pid)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/proc_metrics_utils.py", line 166, in _get_process_and_descendant_metrics
child_processes = process.children(recursive=True)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 336, in wrapper
return fun(self, *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 937, in children
for p in process_iter():
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 1467, in process_iter
if proc.is_running():
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 593, in is_running
return self == Process(self.pid)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 374, in __init__
self._init(pid)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 421, in _init
self._ident = (self.pid, self._create_time)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/__init__.py", line 728, in create_time
return self._create_time
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/_pslinux.py", line 1127, in wrapper
return fun(self, *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/_pslinux.py", line 1301, in create_time
return (float(values[20]) / CLOCK_TICKS) + bt
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/_common.py", line 293, in wrapper
return fun(self)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/psutil/_pslinux.py", line 1170, in _parse_stat_file
return [name] + fields_after_name
# Thread: CredentialManager(140137318008576)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/kt_renewer.py", line 182, in run
self._trigger.wait(_RENEWAL_PERIOD)
File: "/usr/lib64/python2.7/threading.py", line 362, in wait
_sleep(delay)
# Thread: Profile-Plugin(140137292830464)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/__init__.py", line 514, in wrapper
return fn(self, *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/audit/navigator_thread.py", line 169, in _monitor_logs
time.sleep(event_poll_interval)
# Thread: CP Server Thread-7(140137014990592)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-4(140137040168704)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Monitor-GenericMonitor(140135387625216)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: DnsResolutionMonitor(140136411010816)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/stoppable_thread.py", line 35, in run
time.sleep(sleep)
# Thread: Metadata-Plugin(140137301223168)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/__init__.py", line 514, in wrapper
return fn(self, *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/audit/navigator_thread.py", line 169, in _monitor_logs
time.sleep(event_poll_interval)
# Thread: Monitor-HostMonitor(140136419403520)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Thread-15(140135890925312)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/workqueue.py", line 105, in __run_queue
(action, result) = self.queue.get(True)
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Thread-14(140135899318016)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/workqueue.py", line 105, in __run_queue
(action, result) = self.queue.get(True)
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Monitor-GenericMonitor(140135370839808)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Monitor-GenericMonitor(140135379232512)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Thread-17(140135874139904)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/workqueue.py", line 105, in __run_queue
(action, result) = self.queue.get(True)
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-8(140136461367040)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-6(140137023383296)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: Thread-13(140135907710720)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/threadpool.py", line 147, in run
request = self._requests_queue.get(True, self._poll_timeout)
File: "/usr/lib64/python2.7/Queue.py", line 177, in get
self.not_empty.wait(remaining)
File: "/usr/lib64/python2.7/threading.py", line 362, in wait
_sleep(delay)
# Thread: MonitorDaemon-Scheduler(140135916103424)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 362, in wait
_sleep(delay)
# Thread: Monitor-GenericMonitor(140135362447104)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/monitor/wakeable_thread.py", line 35, in run
self._cv.wait(wait_time)
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-5(140137031776000)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-11(140136436188928)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: _TimeoutMonitor(140137065346816)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cherrypy/process/plugins.py", line 515, in run
time.sleep(self.interval)
# Thread: CP Server Thread-10(140136444581632)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-12(140136427796224)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: HTTPServer Thread-2(140137056954112)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cherrypy/process/servers.py", line 225, in _start_http_thread
self.httpserver.start()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/server.py", line 1339, in start
self.tick()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/server.py", line 1461, in tick
return
File: "/usr/lib64/python2.7/socket.py", line 202, in accept
sock, addr = self._sock.accept()
# Thread: Audit-Plugin(140137309615872)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/__init__.py", line 514, in wrapper
return fn(self, *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/audit/navigator_thread.py", line 169, in _monitor_logs
time.sleep(event_poll_interval)
# Thread: CP Server Thread-9(140136452974336)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: MainThread(140137612629824)
File: "/opt/cloudera/cm-agent/bin/cm", line 11, in <module>
load_entry_point('cmf==6.3.1', 'console_scripts', 'cm')()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/main.py", line 317, in main
root(obj=argparse.Namespace())
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 716, in __call__
return self.main(*args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 696, in main
rv = self.invoke(ctx)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 1060, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 1037, in invoke
return Command.invoke(self, ctx)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 889, in invoke
return ctx.invoke(self.callback, **ctx.params)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/core.py", line 534, in invoke
return callback(*args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/main.py", line 129, in agent
main_impl(ctx.obj)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/main.py", line 107, in main_impl
ag.start(legacy_supervisor)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/agent.py", line 883, in start
count=1)
File: "/usr/lib64/python2.7/asyncore.py", line 220, in loop
poll_fun(timeout, map)
File: "/usr/lib64/python2.7/asyncore.py", line 192, in poll2
r = pollster.poll(timeout)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/__init__.py", line 199, in dumpstacks
for filename, lineno, name, line in traceback.extract_stack(stack):
# Thread: Thread-16(140135882532608)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cmf/util/workqueue.py", line 105, in __run_queue
(action, result) = self.queue.get(True)
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
# Thread: CP Server Thread-3(140137048561408)
File: "/usr/lib64/python2.7/threading.py", line 785, in __bootstrap
self.__bootstrap_inner()
File: "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File: "/opt/cloudera/cm-agent/lib/python2.7/site-packages/cheroot/workers/threadpool.py", line 94, in run
conn = self.server.requests.get()
File: "/usr/lib64/python2.7/Queue.py", line 168, in get
self.not_empty.wait()
File: "/usr/lib64/python2.7/threading.py", line 339, in wait
waiter.acquire()
The agent log and the service log running on the server have been deleted, and the fault remains.
How can I solve this problem
Services on the server: hdase yarn impala datanode
