NiFi custom processor custom log, logging in two places

kaliyugantagoni — Thu, 22 Jun 2017 19:47:26 GMT

NiFi 1.2.0

I am having a custom processor and I wish to have a dedicated log file for the same. Accordingly, I have configured the com.datalake.processors.SQLServerCDCProcessor class to use an appender named 'SQLSERVER-CDC'

Following is the logback.xml :

<?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="30 seconds">
    <contextListener>
        <resetJUL>true</resetJUL>
    </contextListener>
    <appender name="APP_FILE">
        <file>/var/log/nifi/nifi-app.log</file>
        <rollingPolicy>
            <!--
                               For daily rollover, use 'app_%d.log'.
              For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
              To GZIP rolled files, replace '.log' with '.log.gz'.
              To ZIP rolled files, replace '.log' with '.log.zip'.
            -->
            <fileNamePattern>/var/log/nifi/archive/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy>
                <maxFileSize>100MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
            <!-- keep 30 log files worth of history -->
            <maxHistory>3</maxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
        </encoder>
     <immediateFlush>true</immediateFlush>
    </appender>
    <appender name="USER_FILE">
        <file>/var/log/nifi/nifi-user.log</file>
        <rollingPolicy>
            <!--
                               For daily rollover, use 'user_%d.log'.
              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
              To GZIP rolled files, replace '.log' with '.log.gz'.
              To ZIP rolled files, replace '.log' with '.log.zip'.
            -->
            <fileNamePattern>/var/log/nifi/archive/nifi-user_%d.log</fileNamePattern>
            <!-- keep 30 log files worth of history -->
            <maxHistory>3</maxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
        </encoder>
    </appender>
    <appender name="BOOTSTRAP_FILE">
        <file>/var/log/nifi/nifi-bootstrap.log</file>
        <rollingPolicy>
            <!--
              For daily rollover, use 'user_%d.log'.
              For hourly rollover, use 'user_%d{yyyy-MM-dd_HH}.log'.
              To GZIP rolled files, replace '.log' with '.log.gz'.
              To ZIP rolled files, replace '.log' with '.log.zip'.
            -->
            <fileNamePattern>/var/log/nifi/archive/nifi-bootstrap_%d.log</fileNamePattern>
            <!-- keep 5 log files worth of history -->
            <maxHistory>5</maxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
        </encoder>
    </appender>
    <appender name="CONSOLE">
        <encoder>
            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
        </encoder>
    </appender>
        <!-- Start : Added for log for custom processor -->
        <appender name="SQLSERVER-CDC">
        <file>/var/log/nifi/sqlserver-cdc.log</file>
        <rollingPolicy>
            <!--
              For daily rollover, use 'app_%d.log'.
              For hourly rollover, use 'app_%d{yyyy-MM-dd_HH}.log'.
              To GZIP rolled files, replace '.log' with '.log.gz'.
              To ZIP rolled files, replace '.log' with '.log.zip'.
            -->
            <fileNamePattern>/var/log/nifi/archive/sqlserver-cdc_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
            <timeBasedFileNamingAndTriggeringPolicy>
                <maxFileSize>25MB</maxFileSize>
            </timeBasedFileNamingAndTriggeringPolicy>
            <!-- keep 30 log files worth of history -->
            <maxHistory>3</maxHistory>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
            <immediateFlush>true</immediateFlush>
        </encoder>
    </appender>
        <!-- End : Added for log for custom processor -->
    <!-- valid logging levels: TRACE, DEBUG, INFO, WARN, ERROR -->
    <logger name="org.apache.nifi" level="INFO"/>
    <logger name="org.apache.nifi.processors" level="WARN"/>
    <logger name="org.apache.nifi.processors.standard.LogAttribute" level="INFO"/>
    <logger name="org.apache.nifi.controller.repository.StandardProcessSession" level="WARN" />
    <logger name="org.apache.zookeeper.ClientCnxn" level="ERROR" />
    <logger name="org.apache.zookeeper.server.NIOServerCnxn" level="ERROR" />
    <logger name="org.apache.zookeeper.server.NIOServerCnxnFactory" level="ERROR" />
    <logger name="org.apache.zookeeper.server.quorum" level="ERROR" />
    <logger name="org.apache.zookeeper.ZooKeeper" level="ERROR" />
    <logger name="org.apache.zookeeper.server.PrepRequestProcessor" level="ERROR" />
    <logger name="org.apache.calcite.runtime.CalciteException" level="OFF" />
    <logger name="org.apache.curator.framework.recipes.leader.LeaderSelector" level="OFF" />
    <logger name="org.apache.curator.ConnectionState" level="OFF" />
    <!-- Logger for managing logging statements for nifi clusters. -->
    <logger name="org.apache.nifi.cluster" level="INFO"/>
    <!-- Logger for logging HTTP requests received by the web server. -->
    <logger name="org.apache.nifi.server.JettyServer" level="INFO"/>
    <!-- Logger for managing logging statements for jetty -->
    <logger name="org.eclipse.jetty" level="INFO"/>
    <!-- Suppress non-error messages due to excessive logging by class or library -->
    <logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR"/>
    <logger name="com.sun.jersey.spi.spring" level="ERROR"/>
    <logger name="org.springframework" level="ERROR"/>
    <!-- Suppress non-error messages due to known warning about redundant path annotation (NIFI-574) -->
    <logger name="com.sun.jersey.spi.inject.Errors" level="ERROR"/>
    <!--
        Logger for capturing user events. We do not want to propagate these
        log events to the root logger. These messages are only sent to the
        user-log appender.
    -->
    <logger name="org.apache.nifi.web.security" level="INFO" additivity="false">
        <appender-ref ref="USER_FILE"/>
    </logger>
    <logger name="org.apache.nifi.web.api.config" level="INFO" additivity="false">
        <appender-ref ref="USER_FILE"/>
    </logger>
    <logger name="org.apache.nifi.authorization" level="INFO" additivity="false">
        <appender-ref ref="USER_FILE"/>
    </logger>
    <logger name="org.apache.nifi.cluster.authorization" level="INFO" additivity="false">
        <appender-ref ref="USER_FILE"/>
    </logger>
    <logger name="org.apache.nifi.web.filter.RequestLogger" level="INFO" additivity="false">
        <appender-ref ref="USER_FILE"/>
    </logger>
    <!--
        Logger for capturing Bootstrap logs and NiFi's standard error and standard out.
    -->
    <logger name="org.apache.nifi.bootstrap" level="INFO" additivity="false">
        <appender-ref ref="BOOTSTRAP_FILE" />
    </logger>
    <logger name="org.apache.nifi.bootstrap.Command" level="INFO" additivity="false">
        <appender-ref ref="CONSOLE" />
        <appender-ref ref="BOOTSTRAP_FILE" />
    </logger>
    <!-- Everything written to NiFi's Standard Out will be logged with the logger org.apache.nifi.StdOut at INFO level -->
    <logger name="org.apache.nifi.StdOut" level="INFO" additivity="false">
        <appender-ref ref="BOOTSTRAP_FILE" />
    </logger>
    <!-- Everything written to NiFi's Standard Error will be logged with the logger org.apache.nifi.StdErr at ERROR level -->
    <logger name="org.apache.nifi.StdErr" level="ERROR" additivity="false">
        <appender-ref ref="BOOTSTRAP_FILE" />
    </logger>
    <!-- Start : Added for log for custom processor -->
    <logger name="com.datalake.processors.SQLServerCDCProcessor" level="DEBUG" >
        <appender-ref ref="SQLSERVER-CDC"/>
    </logger>
    <!-- End : Added for log for custom processor -->
    <root level="INFO">
        <appender-ref ref="APP_FILE"/>
    </root>
</configuration>

The strange fact is that the custom processor debug statements are written to both 'nifi-app.log' and the 'sqlserver-cdc.log' but I want these statements to be written only in the latter('sqlserver-cdc.log').

What am I missing ?

Re: NiFi custom processor custom log, logging in two places

kaliyugantagoni — Wed, 28 Jun 2017 19:24:16 GMT

additivity="false" is essential

Complete answer on StackOverflow.

question Re: NiFi custom processor custom log, logging in two places in Archives of Support Questions (Read Only)

NiFi custom processor custom log, logging in two places

Re: NiFi custom processor custom log, logging in two places