Member since
02-21-2024
7
Posts
4
Kudos Received
0
Solutions
05-28-2024
12:17 AM
Hi @david2658 , I have tried the <scope>provided</scope> previously in custom processor POM which gave a build error but when I have added the same in nar POM the build was successful, however this resulted in a new error at NiFi start. Here is the error log o.e.jetty.ee10.webapp.WebAppContext Failed startup of context oeje10w.WebAppContext@f9b8129{nifi-api,/nifi-api,b=file:///mnt/Development/Workspace/nifi/nifi-binaries/2.0.0-M3/work/jetty/nifi-web-api-2.0.0-M3.war/webapp/,a=AVAILABLE,h=oeje10s.SessionHandler@14485747{STARTED}}{./work/nar/extensions/nifi-server-nar-2.0.0-M3.nar-unpacked/NAR-INF/bundled-dependencies/nifi-web-api-2.0.0-M3.war}
java.lang.NoClassDefFoundError: org/apache/nifi/kafka/shared/property/provider/KafkaPropertyProvider
at java.base/java.lang.Class.forName0(Native Method)
at java.base/java.lang.Class.forName(Class.java:534)
at java.base/java.lang.Class.forName(Class.java:513)
at org.apache.nifi.nar.StandardExtensionDiscoveringManager.getTempComponent(StandardExtensionDiscoveringManager.java:737)
at org.apache.nifi.nar.StandardExtensionDiscoveringManager.isInstanceClassLoaderRequired(StandardExtensionDiscoveringManager.java:475)
at org.apache.nifi.nar.StandardExtensionDiscoveringManager.createInstanceClassLoader(StandardExtensionDiscoveringManager.java:505)
at org.apache.nifi.controller.ExtensionBuilder.createLoggableComponent(ExtensionBuilder.java:915)
at org.apache.nifi.controller.ExtensionBuilder.createLoggableProcessor(ExtensionBuilder.java:766)
at org.apache.nifi.controller.ExtensionBuilder.buildProcessor(ExtensionBuilder.java:251)
at org.apache.nifi.controller.flow.StandardFlowManager.createProcessor(StandardFlowManager.java:359)
at org.apache.nifi.controller.flow.AbstractFlowManager.createProcessor(AbstractFlowManager.java:401)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.addProcessor(StandardVersionedComponentSynchronizer.java:2435)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronizeProcessors(StandardVersionedComponentSynchronizer.java:1042)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:453)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.addProcessGroup(StandardVersionedComponentSynchronizer.java:1212)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronizeChildGroups(StandardVersionedComponentSynchronizer.java:545)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:447)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.lambda$synchronize$0(StandardVersionedComponentSynchronizer.java:248)
at org.apache.nifi.controller.flow.AbstractFlowManager.withParameterContextResolution(AbstractFlowManager.java:638)
at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:243)
at org.apache.nifi.groups.StandardProcessGroup.synchronizeFlow(StandardProcessGroup.java:3868)
at org.apache.nifi.controller.serialization.VersionedFlowSynchronizer.synchronizeFlow(VersionedFlowSynchronizer.java:464)
at org.apache.nifi.controller.serialization.VersionedFlowSynchronizer.sync(VersionedFlowSynchronizer.java:223)
at org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1743)
at org.apache.nifi.persistence.StandardFlowConfigurationDAO.load(StandardFlowConfigurationDAO.java:91)
at org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:805)
at org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:526)
at org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:67)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:497)
at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:670)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
at org.eclipse.jetty.ee10.webapp.WebAppContext.startWebapp(WebAppContext.java:1342)
at org.eclipse.jetty.ee10.webapp.WebAppContext.startContext(WebAppContext.java:1300)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1047)
at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1237)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1044)
at org.eclipse.jetty.ee10.webapp.WebAppContext.doStart(WebAppContext.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.server.Server.start(Server.java:622)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
at org.eclipse.jetty.server.Server.doStart(Server.java:563)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:781)
at org.apache.nifi.NiFi.<init>(NiFi.java:172)
at org.apache.nifi.NiFi.<init>(NiFi.java:83)
at org.apache.nifi.NiFi.main(NiFi.java:332)
Caused by: java.lang.ClassNotFoundException: org.apache.nifi.kafka.shared.property.provider.KafkaPropertyProvider
at java.base/java.net.URLClassLoader.findClass(URLClassLoader.java:445)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:593)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:526)
... 57 common frames omitted
... View more
05-24-2024
04:39 AM
1 Kudo
Hi Cloudera, I am developing a custom processor which uses nifi inbuilt processors as dependencies in pom.xml. This is creating a duplicate processor of existing nifi standard processors with my custom nar version in the NiFi(Please refer to the attachment picture). Is there a way to include these dependencies without bundling them with my custom processor nar. I have tried added the dependencies as `provided` but that is causing the build to fail. I want these dependencies to be referenced from the lib folder where I have placed my nar file. Here are my pom.xml files 1. Main root pom.xml <?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-extension-bundles</artifactId>
<version>2.0.0-M3</version>
</parent>
<groupId>com.test.nifi</groupId>
<artifactId>test-custom-processor-2.X</artifactId>
<version>1.0</version>
<packaging>pom</packaging>
<modules>
<module>nifi-test-processors</module>
<module>nifi-test-nar</module>
</modules>
</project> 2. Nar pom.xml <?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.test.nifi</groupId>
<artifactId>test-custom-processor-2.X</artifactId>
<version>1.0</version>
</parent>
<artifactId>nifi-test-nar</artifactId>
<version>1.0</version>
<packaging>nar</packaging>
<dependencies>
<dependency>
<groupId>com.test.nifi</groupId>
<artifactId>nifi-test-processors</artifactId>
<version>1.0</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-services-api-nar</artifactId>
<version>2.0.0-M3</version>
<type>nar</type>
</dependency>
</dependencies>
</project> 3. My Custom processor pom.xml <?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.test.nifi</groupId>
<artifactId>test-custom-processor-2.X</artifactId>
<version>1.0</version>
</parent>
<artifactId>nifi-test-processors</artifactId>
<packaging>jar</packaging>
<properties>
<java.version>21</java.version>
<nifi.version>2.0.0-M3</nifi.version>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>${nifi.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-oauth2-provider-api</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-proxy-configuration-api</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-standard-processors</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-ssl-context-service-api</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-property-utils</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kafka-2-6-processors</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-kafka-shared</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-record-serialization-service-api</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils-api</artifactId>
<version>${nifi.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.10.1</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-mock</artifactId>
<version>${nifi.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project> Could someone help me setup my custom processor to not include another copy of the dependency processors into my nar file.
... View more
Labels:
- Labels:
-
Apache NiFi
02-22-2024
08:48 PM
No error when I write the flow file but in the super onTrigger session.get() returns a null. So I have tried to transfer to the same queue using session.transfer(ff) but that gives an error saying Cannot transfer FlowFiles that are created in this Session back to self`. Finally I have decided to write a custom processor from scratch using the code from invokeHttp for my usecase. Thank you all for the inputs.
... View more
02-22-2024
02:00 AM
1 Kudo
Hi @SAMSAL , You are correct, session.get() in the parent onTrigger gets the null value. I have tried writing the flowfile but that did not work. I have tried to call the transfer method but got an error saying `java.lang.IllegalArgumentException: Cannot transfer FlowFiles that are created in this Session back to self` Here is the onTrigger code @Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
String memberId = StringUtils.trimToEmpty(context.getProperty("ID").evaluateAttributeExpressions().getValue());
String uri = StringUtils.trimToEmpty(context.getProperty("HTTP URL").evaluateAttributeExpressions().getValue());
getLogger().info("ID: " + memberId + " URI Found: " + uri);
FlowFile requestFlowFile = session.get();
if (requestFlowFile == null) {
requestFlowFile = session.create();
}
requestFlowFile = session.putAttribute(requestFlowFile, "MEMBER_ID", memberId);
// requestFlowFile = session.write(requestFlowFile, (inputStream, outputStream) -> {
// outputStream.write(inputStream.readAllBytes());
// });
// session.commit();
String uri2 = StringUtils.trimToEmpty(context.getProperty("HTTP URL").evaluateAttributeExpressions(requestFlowFile).getValue());
getLogger().info("ID: " + memberId + " URI Found after: " + uri2);
FlowFile flowFile = session.clone(requestFlowFile);
session.transfer(flowFile);
super.onTrigger(context, session);
} If you think there is anything else, I can try by extending the invokeHttp class that would be helpful, else I will just rewrite the invokeHttp code in my custom processor.
... View more
02-21-2024
08:33 PM
1 Kudo
Thank you very much for your inputs. I will try solutions suggested and see which one suits best for my use case.
... View more
02-21-2024
07:17 AM
Thank you very much for the detailed information. The reason I am creating the new flow is because the session.get returns a null and I need the flow file to have the attributes populated for the expression language in the url. If there is a way to create a dynamic string from the properties in the onTrigger that would be great. Also, this processor would be the first in the flow so I assume there will be no flow file to begin with hence the session.create(). The whole purpose of this processor is to give my non developer team a processor that only needs a couple of properties filled and the rest api will be invoked based on the pre-defined configuration in the custom processor.
... View more
02-21-2024
02:34 AM
1 Kudo
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.demo.processors;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.ReadsAttribute;
import org.apache.nifi.annotation.behavior.ReadsAttributes;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.processor.ProcessSession;
import org.apache.nifi.processor.ProcessorInitializationContext;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.processors.standard.InvokeHTTP;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@Tags({"GetMember"})
@CapabilityDescription("Get a Member details from member collection")
@SeeAlso({})
@ReadsAttributes({@ReadsAttribute(attribute = "", description = "")})
@WritesAttributes({@WritesAttribute(attribute = "", description = "")})
@InputRequirement(InputRequirement.Requirement.INPUT_FORBIDDEN)
public class GetMemberProcessor extends InvokeHTTP {
public static final PropertyDescriptor ID = new PropertyDescriptor
.Builder().name("ID")
.displayName("Id")
.description("Member Id")
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
private static final String CUSTOM_URL = "https://my-url/api/v1";
private static final String GET_MEMBER_PATH = "/members";
private List<PropertyDescriptor> descriptors;
@Override
protected void init(final ProcessorInitializationContext context) {
super.init(context);
List<PropertyDescriptor> invokeDescriptors = super.getSupportedPropertyDescriptors();
descriptors = new ArrayList<>();
if (invokeDescriptors != null) {
for (PropertyDescriptor descriptor : invokeDescriptors) {
if (descriptor.getName().equalsIgnoreCase("HTTP URL")) {
continue;
}
descriptors.add(descriptor);
}
}
PropertyDescriptor HTTP_URL = new PropertyDescriptor.Builder()
.name("HTTP URL")
.description("Get Members URL")
.required(false)
.defaultValue(CUSTOM_URL + GET_MEMBER_PATH + "/${MEMBER_ID}")
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.URL_VALIDATOR)
.build();
descriptors.add(ID);
descriptors.add(HTTP_URL);
descriptors = Collections.unmodifiableList(descriptors);
}
@Override
public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
return descriptors;
}
@OnScheduled
public void onScheduled(final ProcessContext context) {
}
@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
String memberId = StringUtils.trimToEmpty(context.getProperty("ID").evaluateAttributeExpressions().getValue());
String uri = StringUtils.trimToEmpty(context.getProperty("HTTP URL").evaluateAttributeExpressions().getValue());
getLogger().info("ID: " + memberId + " URI Found: " + uri);
FlowFile requestFlowFile = session.get();
if (requestFlowFile == null) {
requestFlowFile = session.create();
}
session.putAttribute(requestFlowFile, "MEMBER_ID", memberId);
String uri2 = StringUtils.trimToEmpty(context.getProperty("HTTP URL").evaluateAttributeExpressions().getValue());
getLogger().info("ID: " + memberId + " URI Found after: " + uri2);
super.onTrigger(context, session);
}
} Hi all, I am new to nifi and trying to create a new custom processor extending the invokeHttp processor. I am trying to write a processor will some default values pre configured in to invokeHttp and ask my team to use this processor to add properties that will be used to update properties in the invokeHttp properties. I am getting the following error while trying to run the custom processor Processing halted: yielding [1 sec] org.apache.nifi.processor.exception.FlowFileHandlingException: StandardFlowFileRecord[uuid=5ff2cede-6663-401e-ba5c-d72e62a00913,claim=,offset=0,name=5ff2cede-6663-401e-ba5c-d72e62a00913,size=0] transfer relationship not specified. This FlowFile was created in this session and was not transferred to any Relationship via ProcessSession.transfer() at org.apache.nifi.controller.repository.StandardProcessSession.validateCommitState(StandardProcessSession.java:261) at org.apache.nifi.controller.repository.StandardProcessSession.checkpoint(StandardProcessSession.java:276) at org.apache.nifi.controller.repository.StandardProcessSession.commit(StandardProcessSession.java:559) at org.apache.nifi.controller.repository.StandardProcessSession.commitAsync(StandardProcessSession.java:514) at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:28) at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1274) at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:244) at org.apache.nifi.controller.scheduling.AbstractTimeBasedSchedulingAgent.lambda$doScheduleOnce$0(AbstractTimeBasedSchedulingAgent.java:59) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583)
... View more
Labels:
- Labels:
-
Apache NiFi