Support Questions

Find answers, ask questions, and share your expertise

Adding NiFi standard processors as dependency in custom processor is introducing duplicate processors

avatar

Hi Cloudera,

I am developing a custom processor which uses nifi inbuilt processors as dependencies in pom.xml. This is creating a duplicate processor of existing nifi standard processors with my custom nar version in the NiFi(Please refer to the attachment picture).
Screenshot from 2024-05-24 16-58-40.png 

Is there a way to include these dependencies without bundling them with my custom processor nar. I have tried added the dependencies as `provided` but that is causing the build to fail. I want these dependencies to be referenced from the lib folder where I have placed my nar file.

Here are my pom.xml files

1. Main root pom.xml 

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>org.apache.nifi</groupId>
        <artifactId>nifi-extension-bundles</artifactId>
        <version>2.0.0-M3</version>
    </parent>

    <groupId>com.test.nifi</groupId>
    <artifactId>test-custom-processor-2.X</artifactId>
    <version>1.0</version>
    <packaging>pom</packaging>

    <modules>
        <module>nifi-test-processors</module>
        <module>nifi-test-nar</module>
    </modules>

</project>

 

2. Nar pom.xml

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>com.test.nifi</groupId>
        <artifactId>test-custom-processor-2.X</artifactId>
        <version>1.0</version>
    </parent>

    <artifactId>nifi-test-nar</artifactId>
    <version>1.0</version>
    <packaging>nar</packaging>

    <dependencies>
        <dependency>
            <groupId>com.test.nifi</groupId>
            <artifactId>nifi-test-processors</artifactId>
            <version>1.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-standard-services-api-nar</artifactId>
            <version>2.0.0-M3</version>
            <type>nar</type>
        </dependency>
    </dependencies>

</project>

 

 3. My Custom processor pom.xml

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>com.test.nifi</groupId>
        <artifactId>test-custom-processor-2.X</artifactId>
        <version>1.0</version>
    </parent>

    <artifactId>nifi-test-processors</artifactId>
    <packaging>jar</packaging>

    <properties>
        <java.version>21</java.version>
        <nifi.version>2.0.0-M3</nifi.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-api</artifactId>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-utils</artifactId>
            <version>${nifi.version}</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-oauth2-provider-api</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-proxy-configuration-api</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-standard-processors</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-ssl-context-service-api</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-property-utils</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-kafka-2-6-processors</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-kafka-shared</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-record-serialization-service-api</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-security-utils-api</artifactId>
            <version>${nifi.version}</version>
        </dependency>
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.10.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.nifi</groupId>
            <artifactId>nifi-mock</artifactId>
            <version>${nifi.version}</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-engine</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>
</project>

 

 Could someone help me setup my custom processor to not include another copy of the dependency processors into my nar file.

2 REPLIES 2

avatar
New Contributor

Hello,

To avoid duplicating NiFi standard processors in your custom NAR file, mark NiFi dependencies as "provided" in your POM files. This prevents them from being bundled into your custom NAR and ensures they reference the versions provided by the NiFi runtime. Update your NAR POM and custom processor POM accordingly, using <scope>provided</scope> for dependencies like nifi-standard-services-api-nar and nifi-standard-processors. This setup will avoid conflicts and duplication in your NiFi instance.

avatar

Hi @david2658 ,

I have tried the <scope>provided</scope> previously in custom processor POM which gave a build error but when I have added the same in nar POM the build was successful, however this resulted in a new error at NiFi start. Here is the error log

o.e.jetty.ee10.webapp.WebAppContext Failed startup of context oeje10w.WebAppContext@f9b8129{nifi-api,/nifi-api,b=file:///mnt/Development/Workspace/nifi/nifi-binaries/2.0.0-M3/work/jetty/nifi-web-api-2.0.0-M3.war/webapp/,a=AVAILABLE,h=oeje10s.SessionHandler@14485747{STARTED}}{./work/nar/extensions/nifi-server-nar-2.0.0-M3.nar-unpacked/NAR-INF/bundled-dependencies/nifi-web-api-2.0.0-M3.war}
java.lang.NoClassDefFoundError: org/apache/nifi/kafka/shared/property/provider/KafkaPropertyProvider
	at java.base/java.lang.Class.forName0(Native Method)
	at java.base/java.lang.Class.forName(Class.java:534)
	at java.base/java.lang.Class.forName(Class.java:513)
	at org.apache.nifi.nar.StandardExtensionDiscoveringManager.getTempComponent(StandardExtensionDiscoveringManager.java:737)
	at org.apache.nifi.nar.StandardExtensionDiscoveringManager.isInstanceClassLoaderRequired(StandardExtensionDiscoveringManager.java:475)
	at org.apache.nifi.nar.StandardExtensionDiscoveringManager.createInstanceClassLoader(StandardExtensionDiscoveringManager.java:505)
	at org.apache.nifi.controller.ExtensionBuilder.createLoggableComponent(ExtensionBuilder.java:915)
	at org.apache.nifi.controller.ExtensionBuilder.createLoggableProcessor(ExtensionBuilder.java:766)
	at org.apache.nifi.controller.ExtensionBuilder.buildProcessor(ExtensionBuilder.java:251)
	at org.apache.nifi.controller.flow.StandardFlowManager.createProcessor(StandardFlowManager.java:359)
	at org.apache.nifi.controller.flow.AbstractFlowManager.createProcessor(AbstractFlowManager.java:401)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.addProcessor(StandardVersionedComponentSynchronizer.java:2435)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronizeProcessors(StandardVersionedComponentSynchronizer.java:1042)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:453)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.addProcessGroup(StandardVersionedComponentSynchronizer.java:1212)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronizeChildGroups(StandardVersionedComponentSynchronizer.java:545)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:447)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.lambda$synchronize$0(StandardVersionedComponentSynchronizer.java:248)
	at org.apache.nifi.controller.flow.AbstractFlowManager.withParameterContextResolution(AbstractFlowManager.java:638)
	at org.apache.nifi.flow.synchronization.StandardVersionedComponentSynchronizer.synchronize(StandardVersionedComponentSynchronizer.java:243)
	at org.apache.nifi.groups.StandardProcessGroup.synchronizeFlow(StandardProcessGroup.java:3868)
	at org.apache.nifi.controller.serialization.VersionedFlowSynchronizer.synchronizeFlow(VersionedFlowSynchronizer.java:464)
	at org.apache.nifi.controller.serialization.VersionedFlowSynchronizer.sync(VersionedFlowSynchronizer.java:223)
	at org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1743)
	at org.apache.nifi.persistence.StandardFlowConfigurationDAO.load(StandardFlowConfigurationDAO.java:91)
	at org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:805)
	at org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:526)
	at org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:67)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:497)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:670)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.webapp.WebAppContext.startWebapp(WebAppContext.java:1342)
	at org.eclipse.jetty.ee10.webapp.WebAppContext.startContext(WebAppContext.java:1300)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1047)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1237)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1044)
	at org.eclipse.jetty.ee10.webapp.WebAppContext.doStart(WebAppContext.java:499)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:622)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:563)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:781)
	at org.apache.nifi.NiFi.<init>(NiFi.java:172)
	at org.apache.nifi.NiFi.<init>(NiFi.java:83)
	at org.apache.nifi.NiFi.main(NiFi.java:332)
Caused by: java.lang.ClassNotFoundException: org.apache.nifi.kafka.shared.property.provider.KafkaPropertyProvider
	at java.base/java.net.URLClassLoader.findClass(URLClassLoader.java:445)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:593)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:526)
	... 57 common frames omitted