now all nodes have the same issue and after running a day , some nodes will disconnect from cluster, I upload node 2 and node 3's dump file and nifi-app.log, who can help me locate the cause, thanks in advance. the primary and coordinate node is p-edi-nifi-003:9443 node3 nifi-app.log: 2021-12-15 09:47:37,851 ERROR [Load-Balanced Client Thread-2] o.a.n.c.queue.SwappablePriorityQueue Updated Size of Queue Unacknowledged from FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] to FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] java.lang.RuntimeException: Cannot create negative queue size at org.apache.nifi.controller.queue.SwappablePriorityQueue.logIfNegative(SwappablePriorityQueue.java:1055) at org.apache.nifi.controller.queue.SwappablePriorityQueue.incrementUnacknowledgedQueueSize(SwappablePriorityQueue.java:1045) at org.apache.nifi.controller.queue.SwappablePriorityQueue.acknowledge(SwappablePriorityQueue.java:451) at org.apache.nifi.controller.queue.clustered.partition.RemoteQueuePartition$1.onTransactionFailed(RemoteQueuePartition.java:172) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:271) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:37,893 ERROR [Load-Balanced Client Thread-3] o.a.n.c.q.c.c.a.n.NioAsyncLoadBalanceClient Failed to communicate with Peer p-edi-nifi-002:9443 java.io.EOFException: Encountered End-of-File when trying to verify with Peer p-edi-nifi-002:9443 whether or not space is available in Connection 124d3a0f-0921-13e8-ffff-ffffd6530ec7 at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.receiveSpaceAvailableResponse(LoadBalanceSession.java:584) at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.communicate(LoadBalanceSession.java:151) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:264) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:37,893 ERROR [Load-Balanced Client Thread-3] o.a.n.c.queue.SwappablePriorityQueue Updated Size of Queue Unacknowledged from FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] to FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] java.lang.RuntimeException: Cannot create negative queue size at org.apache.nifi.controller.queue.SwappablePriorityQueue.logIfNegative(SwappablePriorityQueue.java:1055) at org.apache.nifi.controller.queue.SwappablePriorityQueue.incrementUnacknowledgedQueueSize(SwappablePriorityQueue.java:1045) at org.apache.nifi.controller.queue.SwappablePriorityQueue.acknowledge(SwappablePriorityQueue.java:451) at org.apache.nifi.controller.queue.clustered.partition.RemoteQueuePartition$1.onTransactionFailed(RemoteQueuePartition.java:172) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:271) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:38,153 ERROR [Load-Balanced Client Thread-10] o.a.n.c.q.c.c.a.n.NioAsyncLoadBalanceClient Failed to communicate with Peer p-edi-nifi-002:9443 java.io.EOFException: Encountered End-of-File when trying to verify with Peer p-edi-nifi-002:9443 whether or not space is available in Connection 124d3a0f-0921-13e8-ffff-ffffd6530ec7 at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.receiveSpaceAvailableResponse(LoadBalanceSession.java:584) at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.communicate(LoadBalanceSession.java:151) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:264) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:38,153 ERROR [Load-Balanced Client Thread-10] o.a.n.c.queue.SwappablePriorityQueue Updated Size of Queue Unacknowledged from FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] to FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] java.lang.RuntimeException: Cannot create negative queue size at org.apache.nifi.controller.queue.SwappablePriorityQueue.logIfNegative(SwappablePriorityQueue.java:1055) at org.apache.nifi.controller.queue.SwappablePriorityQueue.incrementUnacknowledgedQueueSize(SwappablePriorityQueue.java:1045) at org.apache.nifi.controller.queue.SwappablePriorityQueue.acknowledge(SwappablePriorityQueue.java:451) at org.apache.nifi.controller.queue.clustered.partition.RemoteQueuePartition$1.onTransactionFailed(RemoteQueuePartition.java:172) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:271) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:38,154 ERROR [Load-Balanced Client Thread-11] o.a.n.c.q.c.c.a.n.NioAsyncLoadBalanceClient Failed to communicate with Peer p-edi-nifi-002:9443 java.io.EOFException: Encountered End-of-File when trying to verify with Peer p-edi-nifi-002:9443 whether or not space is available in Connection 124d3a0f-0921-13e8-ffff-ffffd6530ec7 at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.receiveSpaceAvailableResponse(LoadBalanceSession.java:584) at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.communicate(LoadBalanceSession.java:151) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:264) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:38,154 ERROR [Load-Balanced Client Thread-11] o.a.n.c.queue.SwappablePriorityQueue Updated Size of Queue Unacknowledged from FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] to FlowFile Queue Size[ ActiveQueue=[1, 2424 Bytes], Swap Queue=[0, 0 Bytes], Swap Files=[0], Unacknowledged=[-1, -2424 Bytes] ] java.lang.RuntimeException: Cannot create negative queue size at org.apache.nifi.controller.queue.SwappablePriorityQueue.logIfNegative(SwappablePriorityQueue.java:1055) at org.apache.nifi.controller.queue.SwappablePriorityQueue.incrementUnacknowledgedQueueSize(SwappablePriorityQueue.java:1045) at org.apache.nifi.controller.queue.SwappablePriorityQueue.acknowledge(SwappablePriorityQueue.java:451) at org.apache.nifi.controller.queue.clustered.partition.RemoteQueuePartition$1.onTransactionFailed(RemoteQueuePartition.java:172) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:271) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:47:38,154 ERROR [Load-Balanced Client Thread-8] o.a.n.c.q.c.c.a.n.NioAsyncLoadBalanceClient Failed to communicate with Peer p-edi-nifi-002:9443 java.io.EOFException: Encountered End-of-File when trying to verify with Peer p-edi-nifi-002:9443 whether or not space is available in Connection 124d3a0f-0921-13e8-ffff-ffffd6530ec7 at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.receiveSpaceAvailableResponse(LoadBalanceSession.java:584) at org.apache.nifi.controller.queue.clustered.client.async.nio.LoadBalanceSession.communicate(LoadBalanceSession.java:151) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClient.communicate(NioAsyncLoadBalanceClient.java:264) at org.apache.nifi.controller.queue.clustered.client.async.nio.NioAsyncLoadBalanceClientTask.run(NioAsyncLoadBalanceClientTask.java:81) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:834) node3 dump log: "Timer-Driven Process Thread-235" Id=20239 WAITING on java.util.concurrent.locks.ReentrantLock$NonfairSync@2ed0512e at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:943) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1263) at java.base@11.0.12/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1165) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) "Timer-Driven Process Thread-236" Id=20240 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@7797e692 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@27a30d22 "Timer-Driven Process Thread-237" Id=20241 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@19ddb845 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@3546839b "Timer-Driven Process Thread-238" Id=20242 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@5be285e1 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@7feca0fd "Timer-Driven Process Thread-239" Id=20243 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@50c8d0aa at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@346f2d55 "Timer-Driven Process Thread-24" Id=125 WAITING on java.util.concurrent.locks.ReentrantLock$NonfairSync@2ed0512e at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:943) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1263) at java.base@11.0.12/java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:317) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1165) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) "Timer-Driven Process Thread-240" Id=20244 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@326199cd at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@2de59cd9 "Timer-Driven Process Thread-241" Id=20245 WAITING on java.util.concurrent.locks.ReentrantLock$NonfairSync@2ed0512e at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:917) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1240) at java.base@11.0.12/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:267) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.offer(ScheduledThreadPoolExecutor.java:1100) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:1127) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor.delayedExecute(ScheduledThreadPoolExecutor.java:342) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor.scheduleWithFixedDelay(ScheduledThreadPoolExecutor.java:680) at org.apache.nifi.engine.FlowEngine.scheduleWithFixedDelay(FlowEngine.java:97) at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:150) - waiting on org.apache.nifi.controller.scheduling.LifecycleState@19ddb845 at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@7cfd3756 "Timer-Driven Process Thread-242" Id=20246 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@656f3849 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@ec7bba7   node2 nifi-app.log: 2021-12-15 09:14:59,071 ERROR [Load-Balance Server Thread-14224] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-003.gz.cvte.cn/10.21.46.32:43266 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:14:59,073 ERROR [Load-Balance Server Thread-14220] o.a.n.c.q.c.s.StandardLoadBalanceProtocol Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no connection exists with that ID 2021-12-15 09:14:59,073 ERROR [Load-Balance Server Thread-14221] o.a.n.c.q.c.s.StandardLoadBalanceProtocol Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no connection exists with that ID 2021-12-15 09:14:59,073 ERROR [Load-Balance Server Thread-14220] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-003.gz.cvte.cn/10.21.46.32:43256 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:14:59,073 ERROR [Load-Balance Server Thread-14221] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-003.gz.cvte.cn/10.21.46.32:43262 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:14:59,073 ERROR [Load-Balance Server Thread-14226] o.a.n.c.q.c.s.StandardLoadBalanceProtocol Attempted to receive FlowFiles from Peer p-edi-nifi-005 for Connection with ID b54a33c5-2264-1ded-ffff-ffffbd054c96 but no connection exists with that ID 2021-12-15 09:14:59,074 ERROR [Load-Balance Server Thread-14226] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-005.gz.cvte.cn/10.21.46.34:60262 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-005 for Connection with ID b54a33c5-2264-1ded-ffff-ffffbd054c96 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:14:59,076 ERROR [Load-Balance Server Thread-14218] o.a.n.c.q.c.s.StandardLoadBalanceProtocol Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no connection exists with that ID 2021-12-15 09:14:59,076 ERROR [Load-Balance Server Thread-14218] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-003.gz.cvte.cn/10.21.46.32:43255 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) 2021-12-15 09:14:59,077 ERROR [Load-Balance Server Thread-14223] o.a.n.c.q.c.s.StandardLoadBalanceProtocol Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no connection exists with that ID 2021-12-15 09:14:59,078 ERROR [Load-Balance Server Thread-14223] o.a.n.c.q.c.s.ConnectionLoadBalanceServer Failed to communicate with Peer p-edi-nifi-003.gz.cvte.cn/10.21.46.32:43264 org.apache.nifi.controller.queue.clustered.server.TransactionAbortedException: Attempted to receive FlowFiles from Peer p-edi-nifi-003 for Connection with ID 124d3a0f-0921-13e8-ffff-ffffd6530ec7 but no Connection exists with that ID at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:205) at org.apache.nifi.controller.queue.clustered.server.StandardLoadBalanceProtocol.receiveFlowFiles(StandardLoadBalanceProtocol.java:134) at org.apache.nifi.controller.queue.clustered.server.ConnectionLoadBalanceServer$CommunicateAction.run(ConnectionLoadBalanceServer.java:162) at java.base/java.lang.Thread.run(Thread.java:834) node2 dump file: "StandardHttpContextMap-aca62685-0179-1000-ffff-ffff821b01ae" Id=204 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@57c01834 at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) "StandardHttpContextMap-dd9c9135-0178-1000-0000-0000464c518b" Id=153 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7f1251b8 at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) "SyncThread:2" Id=88 WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7e205cc7 at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081) at java.base@11.0.12/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433) at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:109) "ThreadPoolRequestReplicator Maintenance Thread" Id=122 TIMED_WAITING on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@138779d3 at java.base@11.0.12/jdk.internal.misc.Unsafe.park(Native Method) at java.base@11.0.12/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234) at java.base@11.0.12/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) "Timer-1" Id=20698 WAITING on java.util.TaskQueue@779365a3 at java.base@11.0.12/java.lang.Object.wait(Native Method) at java.base@11.0.12/java.lang.Object.wait(Object.java:328) at java.base@11.0.12/java.util.TimerThread.mainLoop(Timer.java:527) at java.base@11.0.12/java.util.TimerThread.run(Timer.java:506) "Timer-Driven Process Thread-1" Id=76 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@12189489 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@64236be "Timer-Driven Process Thread-10" Id=156 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@3c217039 at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@ca31114 "Timer-Driven Process Thread-100" Id=5721 BLOCKED on org.apache.nifi.controller.scheduling.LifecycleState@70dce42a at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:148) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base@11.0.12/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@11.0.12/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@11.0.12/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.12/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.12/java.lang.Thread.run(Thread.java:834) Number of Locked Synchronizers: 1 - java.util.concurrent.ThreadPoolExecutor$Worker@fdaafba   ... View more