Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

Cannot import data to postgresSQL

Cannot import data to postgresSQL

Explorer

Cannot import data to postgresSQL (SAME issue with PutDatabaseRecord and PutSQL )

 

 due to Batch entry 0 INSERT INTO customer (customer_id, store_id, first_name, last_name, email, address_id, activebool, create_date, last_update, active) VALUES (1,1,'Mary','Smith','mary.smith@sakilacustomer.org',5,'TRUE','2006-02-14','2013-05-26 14:49:45',1) was aborted: ERROR: column "create_date" is of type date but expression is of type character varying

 


csv to postgres.PNG

 

 

Any suggestions? (xml file below)

Spoiler

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<template encoding-version="1.3">
<description></description>
<groupId>a3d73b93-5247-1def-509e-14cd1ac69742</groupId>
<name>CSV to Postgresql</name>
<snippet>
<connections>
<id>4fc6d8b8-bd61-3313-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>4fc6d8b8-bd61-3313-98fc-59d5256a421e</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>7cc91796-c792-3b07-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>7cc91796-c792-3b07-ae1d-8fc97ad5b74a</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>ddd0dff0-76dd-3427-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>ddd0dff0-76dd-3427-852f-d4fa13e9ec46</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>6c54eacd-143c-33fa-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>6c54eacd-143c-33fa-aebc-f9e61ce024f2</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>031921be-9955-326b-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>031921be-9955-326b-b891-5e1c9d5536f4</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>834c6890-31d2-3aed-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>834c6890-31d2-3aed-b6bc-c740e496ce94</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>031921be-9955-326b-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>031921be-9955-326b-b891-5e1c9d5536f4</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>98af5b16-4035-379a-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>98af5b16-4035-379a-ab1c-c5e5d95aabdb</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>8f478363-3718-3893-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>8f478363-3718-3893-9c04-2986d3d46720</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>ddd0dff0-76dd-3427-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>ddd0dff0-76dd-3427-852f-d4fa13e9ec46</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>failure</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>aa3bba4a-8b32-3ca5-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>aa3bba4a-8b32-3ca5-888b-d616b2a53d7c</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>7cc91796-c792-3b07-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>7cc91796-c792-3b07-ae1d-8fc97ad5b74a</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>success</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>cf43fc03-eac8-3164-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>cf43fc03-eac8-3164-8079-e434ae2839bc</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>9f35d6a6-94ae-3849-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>9f35d6a6-94ae-3849-85b7-6115a9fc87fe</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>failure</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>ddd0dff0-76dd-3427-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>ddd0dff0-76dd-3427-852f-d4fa13e9ec46</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<connections>
<id>eaba1dbc-b2a8-30be-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>eaba1dbc-b2a8-30be-afb1-440bdbfba52b</versionedComponentId>
<backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
<backPressureObjectThreshold>10000</backPressureObjectThreshold>
<bends>
<x>470.94140625</x>
<y>357.442138671875</y>
</bends>
<bends>
<x>526.94140625</x>
<y>429.442138671875</y>
</bends>
<destination>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
</destination>
<flowFileExpiration>0 sec</flowFileExpiration>
<labelIndex>1</labelIndex>
<loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression>
<loadBalancePartitionAttribute></loadBalancePartitionAttribute>
<loadBalanceStatus>LOAD_BALANCE_NOT_CONFIGURED</loadBalanceStatus>
<loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy>
<name></name>
<selectedRelationships>retry</selectedRelationships>
<source>
<groupId>bcdb9e55-19ce-3ec3-0000-000000000000</groupId>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<type>PROCESSOR</type>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
</source>
<zIndex>0</zIndex>
</connections>
<controllerServices>
<id>1390d92e-bc8d-3ccd-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>1390d92e-bc8d-3ccd-86ff-93bb5fd79fc1</versionedComponentId>
<bundle>
<artifact>nifi-registry-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<comments></comments>
<descriptors>
<entry>
<key>avro-reg-validated-field-names</key>
<value>
<name>avro-reg-validated-field-names</name>
</value>
</entry>
<entry>
<key>customer</key>
<value>
<name>customer</name>
</value>
</entry>
</descriptors>
<name>potgresCustomerWithoutheader -- AvroSchemaRegistry</name>
<persistsState>false</persistsState>
<properties>
<entry>
<key>avro-reg-validated-field-names</key>
</entry>
<entry>
<key>customer</key>
<value>{ "type":"record",
"name":"customer",
"namespace":"nifi",
"fields":[
{"name":"customer_id","type":["null","int"]},
{"name":"store_id","type":["null","int"]},
{"name":"first_name","type":["null","string"]},
{"name":"last_name","type":["null","string"]},
{"name":"email","type":["null","string"]},
{"name":"address_id","type":["null","int"]},
{"name":"activebool","type":["null","boolean"]},
{"name":"create_date","type":"string"},
{"name":"last_update","type":["null","string"]},
{"name":"active","type":["null","int"]}
]}</value>
</entry>
</properties>
<state>ENABLED</state>
<type>org.apache.nifi.schemaregistry.services.AvroSchemaRegistry</type>
</controllerServices>
<controllerServices>
<id>868992b5-01dc-3f11-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<versionedComponentId>868992b5-01dc-3f11-bb43-23b61ed20a90</versionedComponentId>
<bundle>
<artifact>nifi-dbcp-service-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<comments></comments>
<descriptors>
<entry>
<key>Database Connection URL</key>
<value>
<name>Database Connection URL</name>
</value>
</entry>
<entry>
<key>Database Driver Class Name</key>
<value>
<name>Database Driver Class Name</name>
</value>
</entry>
<entry>
<key>database-driver-locations</key>
<value>
<name>database-driver-locations</name>
</value>
</entry>
<entry>
<key>kerberos-credentials-service</key>
<value>
<identifiesControllerService>org.apache.nifi.kerberos.KerberosCredentialsService</identifiesControllerService>
<name>kerberos-credentials-service</name>
</value>
</entry>
<entry>
<key>Database User</key>
<value>
<name>Database User</name>
</value>
</entry>
<entry>
<key>Password</key>
<value>
<name>Password</name>
</value>
</entry>
<entry>
<key>Max Wait Time</key>
<value>
<name>Max Wait Time</name>
</value>
</entry>
<entry>
<key>Max Total Connections</key>
<value>
<name>Max Total Connections</name>
</value>
</entry>
<entry>
<key>Validation-query</key>
<value>
<name>Validation-query</name>
</value>
</entry>
<entry>
<key>dbcp-min-idle-conns</key>
<value>
<name>dbcp-min-idle-conns</name>
</value>
</entry>
<entry>
<key>dbcp-max-idle-conns</key>
<value>
<name>dbcp-max-idle-conns</name>
</value>
</entry>
<entry>
<key>dbcp-max-conn-lifetime</key>
<value>
<name>dbcp-max-conn-lifetime</name>
</value>
</entry>
<entry>
<key>dbcp-time-between-eviction-runs</key>
<value>
<name>dbcp-time-between-eviction-runs</name>
</value>
</entry>
<entry>
<key>dbcp-min-evictable-idle-time</key>
<value>
<name>dbcp-min-evictable-idle-time</name>
</value>
</entry>
<entry>
<key>dbcp-soft-min-evictable-idle-time</key>
<value>
<name>dbcp-soft-min-evictable-idle-time</name>
</value>
</entry>
</descriptors>
<name>postgres -- DBCPConnectionPool</name>
<persistsState>false</persistsState>
<properties>
<entry>
<key>Database Connection URL</key>
<value>jdbc:postgresql://postgresdb.ckdsaqpaaule.us-east-1.rds.amazonaws.com:5432/thuyle_dvdrental</value>
</entry>
<entry>
<key>Database Driver Class Name</key>
<value>org.postgresql.Driver</value>
</entry>
<entry>
<key>database-driver-locations</key>
<value>/home/nifi/jdbc-drivers/postgresql-42.2.6.jar</value>
</entry>
<entry>
<key>kerberos-credentials-service</key>
</entry>
<entry>
<key>Database User</key>
<value>postgresadmin</value>
</entry>
<entry>
<key>Password</key>
</entry>
<entry>
<key>Max Wait Time</key>
</entry>
<entry>
<key>Max Total Connections</key>
</entry>
<entry>
<key>Validation-query</key>
</entry>
<entry>
<key>dbcp-min-idle-conns</key>
</entry>
<entry>
<key>dbcp-max-idle-conns</key>
</entry>
<entry>
<key>dbcp-max-conn-lifetime</key>
</entry>
<entry>
<key>dbcp-time-between-eviction-runs</key>
</entry>
<entry>
<key>dbcp-min-evictable-idle-time</key>
</entry>
<entry>
<key>dbcp-soft-min-evictable-idle-time</key>
</entry>
</properties>
<state>ENABLED</state>
<type>org.apache.nifi.dbcp.DBCPConnectionPool</type>
</controllerServices>
<controllerServices>
<id>e6a5d1b7-12c8-3106-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<bundle>
<artifact>nifi-record-serialization-services-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<comments></comments>
<descriptors>
<entry>
<key>schema-access-strategy</key>
<value>
<name>schema-access-strategy</name>
</value>
</entry>
<entry>
<key>schema-registry</key>
<value>
<identifiesControllerService>org.apache.nifi.schemaregistry.services.SchemaRegistry</identifiesControllerService>
<name>schema-registry</name>
</value>
</entry>
<entry>
<key>schema-name</key>
<value>
<name>schema-name</name>
</value>
</entry>
<entry>
<key>schema-version</key>
<value>
<name>schema-version</name>
</value>
</entry>
<entry>
<key>schema-branch</key>
<value>
<name>schema-branch</name>
</value>
</entry>
<entry>
<key>schema-text</key>
<value>
<name>schema-text</name>
</value>
</entry>
<entry>
<key>csv-reader-csv-parser</key>
<value>
<name>csv-reader-csv-parser</name>
</value>
</entry>
<entry>
<key>Date Format</key>
<value>
<name>Date Format</name>
</value>
</entry>
<entry>
<key>Time Format</key>
<value>
<name>Time Format</name>
</value>
</entry>
<entry>
<key>Timestamp Format</key>
<value>
<name>Timestamp Format</name>
</value>
</entry>
<entry>
<key>CSV Format</key>
<value>
<name>CSV Format</name>
</value>
</entry>
<entry>
<key>Value Separator</key>
<value>
<name>Value Separator</name>
</value>
</entry>
<entry>
<key>Skip Header Line</key>
<value>
<name>Skip Header Line</name>
</value>
</entry>
<entry>
<key>ignore-csv-header</key>
<value>
<name>ignore-csv-header</name>
</value>
</entry>
<entry>
<key>Quote Character</key>
<value>
<name>Quote Character</name>
</value>
</entry>
<entry>
<key>Escape Character</key>
<value>
<name>Escape Character</name>
</value>
</entry>
<entry>
<key>Comment Marker</key>
<value>
<name>Comment Marker</name>
</value>
</entry>
<entry>
<key>Null String</key>
<value>
<name>Null String</name>
</value>
</entry>
<entry>
<key>Trim Fields</key>
<value>
<name>Trim Fields</name>
</value>
</entry>
<entry>
<key>csvutils-character-set</key>
<value>
<name>csvutils-character-set</name>
</value>
</entry>
</descriptors>
<name>potgresCustomerWithoutheader -- CSVReader</name>
<persistsState>false</persistsState>
<properties>
<entry>
<key>schema-access-strategy</key>
<value>schema-name</value>
</entry>
<entry>
<key>schema-registry</key>
<value>1390d92e-bc8d-3ccd-0000-000000000000</value>
</entry>
<entry>
<key>schema-name</key>
</entry>
<entry>
<key>schema-version</key>
</entry>
<entry>
<key>schema-branch</key>
</entry>
<entry>
<key>schema-text</key>
</entry>
<entry>
<key>csv-reader-csv-parser</key>
</entry>
<entry>
<key>Date Format</key>
<value>yyyy-MM-dd</value>
</entry>
<entry>
<key>Time Format</key>
<value>HH:mm:ss</value>
</entry>
<entry>
<key>Timestamp Format</key>
<value>yyyy-MM-dd HH:mm:ss</value>
</entry>
<entry>
<key>CSV Format</key>
</entry>
<entry>
<key>Value Separator</key>
</entry>
<entry>
<key>Skip Header Line</key>
</entry>
<entry>
<key>ignore-csv-header</key>
</entry>
<entry>
<key>Quote Character</key>
</entry>
<entry>
<key>Escape Character</key>
</entry>
<entry>
<key>Comment Marker</key>
</entry>
<entry>
<key>Null String</key>
</entry>
<entry>
<key>Trim Fields</key>
</entry>
<entry>
<key>csvutils-character-set</key>
</entry>
</properties>
<state>ENABLED</state>
<type>org.apache.nifi.csv.CSVReader</type>
</controllerServices>
<processors>
<id>031921be-9955-326b-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>8.0</x>
<y>224.0</y>
</position>
<versionedComponentId>031921be-9955-326b-b891-5e1c9d5536f4</versionedComponentId>
<bundle>
<artifact>nifi-update-attribute-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Delete Attributes Expression</key>
<value>
<name>Delete Attributes Expression</name>
</value>
</entry>
<entry>
<key>Store State</key>
<value>
<name>Store State</name>
</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
<value>
<name>Stateful Variables Initial Value</name>
</value>
</entry>
<entry>
<key>canonical-value-lookup-cache-size</key>
<value>
<name>canonical-value-lookup-cache-size</name>
</value>
</entry>
<entry>
<key>error_file</key>
<value>
<name>error_file</name>
</value>
</entry>
<entry>
<key>received_date</key>
<value>
<name>received_date</name>
</value>
</entry>
<entry>
<key>running_count</key>
<value>
<name>running_count</name>
</value>
</entry>
<entry>
<key>schema.name</key>
<value>
<name>schema.name</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Delete Attributes Expression</key>
</entry>
<entry>
<key>Store State</key>
<value>Do not store state</value>
</entry>
<entry>
<key>Stateful Variables Initial Value</key>
</entry>
<entry>
<key>canonical-value-lookup-cache-size</key>
<value>100</value>
</entry>
<entry>
<key>error_file</key>
<value>/home/nifi/error_customer.csv</value>
</entry>
<entry>
<key>received_date</key>
<value>${now():format("yyyy-MM-dd HH:mm:ss.SSSS")}</value>
</entry>
<entry>
<key>running_count</key>
<value>0</value>
</entry>
<entry>
<key>schema.name</key>
<value>customer</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>UpdateAttribute</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.attributes.UpdateAttribute</type>
</processors>
<processors>
<id>4632754b-e43b-3b79-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>448.0</y>
</position>
<versionedComponentId>4632754b-e43b-3b79-b472-df88a152ce2e</versionedComponentId>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>put-db-record-record-reader</key>
<value>
<identifiesControllerService>org.apache.nifi.serialization.RecordReaderFactory</identifiesControllerService>
<name>put-db-record-record-reader</name>
</value>
</entry>
<entry>
<key>put-db-record-statement-type</key>
<value>
<name>put-db-record-statement-type</name>
</value>
</entry>
<entry>
<key>put-db-record-dcbp-service</key>
<value>
<identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService>
<name>put-db-record-dcbp-service</name>
</value>
</entry>
<entry>
<key>put-db-record-catalog-name</key>
<value>
<name>put-db-record-catalog-name</name>
</value>
</entry>
<entry>
<key>put-db-record-schema-name</key>
<value>
<name>put-db-record-schema-name</name>
</value>
</entry>
<entry>
<key>put-db-record-table-name</key>
<value>
<name>put-db-record-table-name</name>
</value>
</entry>
<entry>
<key>put-db-record-translate-field-names</key>
<value>
<name>put-db-record-translate-field-names</name>
</value>
</entry>
<entry>
<key>put-db-record-unmatched-field-behavior</key>
<value>
<name>put-db-record-unmatched-field-behavior</name>
</value>
</entry>
<entry>
<key>put-db-record-unmatched-column-behavior</key>
<value>
<name>put-db-record-unmatched-column-behavior</name>
</value>
</entry>
<entry>
<key>put-db-record-update-keys</key>
<value>
<name>put-db-record-update-keys</name>
</value>
</entry>
<entry>
<key>put-db-record-field-containing-sql</key>
<value>
<name>put-db-record-field-containing-sql</name>
</value>
</entry>
<entry>
<key>put-db-record-allow-multiple-statements</key>
<value>
<name>put-db-record-allow-multiple-statements</name>
</value>
</entry>
<entry>
<key>put-db-record-quoted-identifiers</key>
<value>
<name>put-db-record-quoted-identifiers</name>
</value>
</entry>
<entry>
<key>put-db-record-quoted-table-identifiers</key>
<value>
<name>put-db-record-quoted-table-identifiers</name>
</value>
</entry>
<entry>
<key>put-db-record-query-timeout</key>
<value>
<name>put-db-record-query-timeout</name>
</value>
</entry>
<entry>
<key>rollback-on-failure</key>
<value>
<name>rollback-on-failure</name>
</value>
</entry>
<entry>
<key>table-schema-cache-size</key>
<value>
<name>table-schema-cache-size</name>
</value>
</entry>
<entry>
<key>put-db-record-max-batch-size</key>
<value>
<name>put-db-record-max-batch-size</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>put-db-record-record-reader</key>
<value>e6a5d1b7-12c8-3106-0000-000000000000</value>
</entry>
<entry>
<key>put-db-record-statement-type</key>
<value>INSERT</value>
</entry>
<entry>
<key>put-db-record-dcbp-service</key>
<value>868992b5-01dc-3f11-0000-000000000000</value>
</entry>
<entry>
<key>put-db-record-catalog-name</key>
</entry>
<entry>
<key>put-db-record-schema-name</key>
</entry>
<entry>
<key>put-db-record-table-name</key>
<value>customer</value>
</entry>
<entry>
<key>put-db-record-translate-field-names</key>
<value>true</value>
</entry>
<entry>
<key>put-db-record-unmatched-field-behavior</key>
<value>Ignore Unmatched Fields</value>
</entry>
<entry>
<key>put-db-record-unmatched-column-behavior</key>
<value>Fail on Unmatched Columns</value>
</entry>
<entry>
<key>put-db-record-update-keys</key>
</entry>
<entry>
<key>put-db-record-field-containing-sql</key>
</entry>
<entry>
<key>put-db-record-allow-multiple-statements</key>
<value>false</value>
</entry>
<entry>
<key>put-db-record-quoted-identifiers</key>
<value>false</value>
</entry>
<entry>
<key>put-db-record-quoted-table-identifiers</key>
<value>false</value>
</entry>
<entry>
<key>put-db-record-query-timeout</key>
<value>0 seconds</value>
</entry>
<entry>
<key>rollback-on-failure</key>
<value>false</value>
</entry>
<entry>
<key>table-schema-cache-size</key>
<value>100</value>
</entry>
<entry>
<key>put-db-record-max-batch-size</key>
<value>0</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>Postgres-- PutDatabaseRecord</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>retry</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.standard.PutDatabaseRecord</type>
</processors>
<processors>
<id>7cc91796-c792-3b07-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>0.0</x>
<y>672.0</y>
</position>
<versionedComponentId>7cc91796-c792-3b07-ae1d-8fc97ad5b74a</versionedComponentId>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Log Level</key>
<value>
<name>Log Level</name>
</value>
</entry>
<entry>
<key>Log Payload</key>
<value>
<name>Log Payload</name>
</value>
</entry>
<entry>
<key>Attributes to Log</key>
<value>
<name>Attributes to Log</name>
</value>
</entry>
<entry>
<key>attributes-to-log-regex</key>
<value>
<name>attributes-to-log-regex</name>
</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
<value>
<name>Attributes to Ignore</name>
</value>
</entry>
<entry>
<key>attributes-to-ignore-regex</key>
<value>
<name>attributes-to-ignore-regex</name>
</value>
</entry>
<entry>
<key>Log prefix</key>
<value>
<name>Log prefix</name>
</value>
</entry>
<entry>
<key>character-set</key>
<value>
<name>character-set</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Log Level</key>
<value>info</value>
</entry>
<entry>
<key>Log Payload</key>
<value>false</value>
</entry>
<entry>
<key>Attributes to Log</key>
</entry>
<entry>
<key>attributes-to-log-regex</key>
<value>.*</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
</entry>
<entry>
<key>attributes-to-ignore-regex</key>
</entry>
<entry>
<key>Log prefix</key>
</entry>
<entry>
<key>character-set</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>Success-Log</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.LogAttribute</type>
</processors>
<processors>
<id>98af5b16-4035-379a-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>8.0</x>
<y>0.0</y>
</position>
<versionedComponentId>98af5b16-4035-379a-ab1c-c5e5d95aabdb</versionedComponentId>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>File Size</key>
<value>
<name>File Size</name>
</value>
</entry>
<entry>
<key>Batch Size</key>
<value>
<name>Batch Size</name>
</value>
</entry>
<entry>
<key>Data Format</key>
<value>
<name>Data Format</name>
</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>
<name>Unique FlowFiles</name>
</value>
</entry>
<entry>
<key>generate-ff-custom-text</key>
<value>
<name>generate-ff-custom-text</name>
</value>
</entry>
<entry>
<key>character-set</key>
<value>
<name>character-set</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>File Size</key>
<value>0B</value>
</entry>
<entry>
<key>Batch Size</key>
<value>1</value>
</entry>
<entry>
<key>Data Format</key>
<value>Text</value>
</entry>
<entry>
<key>Unique FlowFiles</key>
<value>false</value>
</entry>
<entry>
<key>generate-ff-custom-text</key>
<value>1,1,Mary,Smith,mary.smith@sakilacustomer.org,5,true,2006-02-14,2013-05-26 14:49:45,1
2,1,Patricia,Johnson,patricia.johnson@sakilacustomer.org,6,true,2006-02-14,2013-05-26 14:49:45,1
3,1,Linda,Williams,linda.williams@sakilacustomer.org,7,true,2006-02-14,2013-05-26 14:49:45,1
4,2,Barbara,Jones,barbara.jones@sakilacustomer.org,8,true,2006-02-14,2013-05-26 14:49:45,1
5,1,Elizabeth,Brown,elizabeth.brown@sakilacustomer.org,9,true,2006-02-14,2013-05-26 14:49:45,1
6,2,Jennifer,Davis,jennifer.davis@sakilacustomer.org,10,true,2006-02-14,2013-05-26 14:49:45,1
7,1,Maria,Miller,maria.miller@sakilacustomer.org,11,true,2006-02-14,2013-05-26 14:49:45,1
8,2,Susan,Wilson,susan.wilson@sakilacustomer.org,12,true,2006-02-14,2013-05-26 14:49:45,1
9,2,Margaret,Moore,margaret.moore@sakilacustomer.org,13,true,2006-02-14,2013-05-26 14:49:45,1
10,1,Dorothy,Taylor,dorothy.taylor@sakilacustomer.org,14,true,2006-02-14,2013-05-26 14:49:45,1
11,2,Lisa,Anderson,lisa.anderson@sakilacustomer.org,15,true,2006-02-14,2013-05-26 14:49:45,1
12,1,Nancy,Thomas,nancy.thomas@sakilacustomer.org,16,true,2006-02-14,2013-05-26 14:49:45,1
13,2,Karen,Jackson,karen.jackson@sakilacustomer.org,17,true,2006-02-14,2013-05-26 14:49:45,1
14,2,Betty,White,betty.white@sakilacustomer.org,18,true,2006-02-14,2013-05-26 14:49:45,1
15,1,Helen,Harris,helen.harris@sakilacustomer.org,19,true,2006-02-14,2013-05-26 14:49:45,1
16,2,Sandra,Martin,sandra.martin@sakilacustomer.org,20,true,2006-02-14,2013-05-26 14:49:45,0
17,1,Donna,Thompson,donna.thompson@sakilacustomer.org,21,true,2006-02-14,2013-05-26 14:49:45,1
18,2,Carol,Garcia,carol.garcia@sakilacustomer.org,22,true,2006-02-14,2013-05-26 14:49:45,1
19,1,Ruth,Martinez,ruth.martinez@sakilacustomer.org,23,true,2006-02-14,2013-05-26 14:49:45,1
20,2,Sharon,Robinson,sharon.robinson@sakilacustomer.org,24,true,2006-02-14,2013-05-26 14:49:45,1
21,1,Michelle,Clark,michelle.clark@sakilacustomer.org,25,true,2006-02-14,2013-05-26 14:49:45,1
22,1,Laura,Rodriguez,laura.rodriguez@sakilacustomer.org,26,true,2006-02-14,2013-05-26 14:49:45,1</value>
</entry>
<entry>
<key>character-set</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>10000000000 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>without header -- GenerateFlowFile</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>RUNNING</state>
<style/>
<type>org.apache.nifi.processors.standard.GenerateFlowFile</type>
</processors>
<processors>
<id>9f35d6a6-94ae-3849-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>616.0</x>
<y>224.0</y>
</position>
<versionedComponentId>9f35d6a6-94ae-3849-85b7-6115a9fc87fe</versionedComponentId>
<bundle>
<artifact>nifi-standard-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Log Level</key>
<value>
<name>Log Level</name>
</value>
</entry>
<entry>
<key>Log Payload</key>
<value>
<name>Log Payload</name>
</value>
</entry>
<entry>
<key>Attributes to Log</key>
<value>
<name>Attributes to Log</name>
</value>
</entry>
<entry>
<key>attributes-to-log-regex</key>
<value>
<name>attributes-to-log-regex</name>
</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
<value>
<name>Attributes to Ignore</name>
</value>
</entry>
<entry>
<key>attributes-to-ignore-regex</key>
<value>
<name>attributes-to-ignore-regex</name>
</value>
</entry>
<entry>
<key>Log prefix</key>
<value>
<name>Log prefix</name>
</value>
</entry>
<entry>
<key>character-set</key>
<value>
<name>character-set</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Log Level</key>
<value>info</value>
</entry>
<entry>
<key>Log Payload</key>
<value>false</value>
</entry>
<entry>
<key>Attributes to Log</key>
</entry>
<entry>
<key>attributes-to-log-regex</key>
<value>.*</value>
</entry>
<entry>
<key>Attributes to Ignore</key>
</entry>
<entry>
<key>attributes-to-ignore-regex</key>
</entry>
<entry>
<key>Log prefix</key>
</entry>
<entry>
<key>character-set</key>
<value>UTF-8</value>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>Failure-Log</name>
<relationships>
<autoTerminate>true</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.standard.LogAttribute</type>
</processors>
<processors>
<id>ddd0dff0-76dd-3427-0000-000000000000</id>
<parentGroupId>bcdb9e55-19ce-3ec3-0000-000000000000</parentGroupId>
<position>
<x>616.0</x>
<y>456.0</y>
</position>
<versionedComponentId>ddd0dff0-76dd-3427-852f-d4fa13e9ec46</versionedComponentId>
<bundle>
<artifact>nifi-scripting-nar</artifact>
<group>org.apache.nifi</group>
<version>1.11.1</version>
</bundle>
<config>
<bulletinLevel>WARN</bulletinLevel>
<comments></comments>
<concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
<descriptors>
<entry>
<key>Script Engine</key>
<value>
<name>Script Engine</name>
</value>
</entry>
<entry>
<key>Script File</key>
<value>
<name>Script File</name>
</value>
</entry>
<entry>
<key>Script Body</key>
<value>
<name>Script Body</name>
</value>
</entry>
<entry>
<key>Module Directory</key>
<value>
<name>Module Directory</name>
</value>
</entry>
</descriptors>
<executionNode>ALL</executionNode>
<lossTolerant>false</lossTolerant>
<penaltyDuration>30 sec</penaltyDuration>
<properties>
<entry>
<key>Script Engine</key>
<value>python</value>
</entry>
<entry>
<key>Script File</key>
</entry>
<entry>
<key>Script Body</key>
<value>from org.apache.nifi.processors.script import ExecuteScript
import subprocess
from org.apache.commons.io import IOUtils
from java.nio.charset import StandardCharsets
from org.apache.nifi.processor.io import InputStreamCallback
import os.path


# Define a subclass of InputStreamCallback for use in session.read()
class PyInputStreamCallback(InputStreamCallback):
def __init__(self, error_file):
self.error_file = error_file
pass

def process(self, inputStream):
text = IOUtils.toString(inputStream, StandardCharsets.UTF_8)
# Do something with text here
if(text != None):
fd = open(self.error_file,'a')
fd.write(text)
fd.write("\n")
fd.close()
# end class

# implicit return at the end
flowFile = session.get()
if flowFile is not None:
error_file = flowFile.getAttribute('error_file')
session.read(flowFile, PyInputStreamCallback(error_file))
session.transfer(flowFile,ExecuteScript.REL_SUCCESS)


</value>
</entry>
<entry>
<key>Module Directory</key>
</entry>
</properties>
<runDurationMillis>0</runDurationMillis>
<schedulingPeriod>0 sec</schedulingPeriod>
<schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
<yieldDuration>1 sec</yieldDuration>
</config>
<executionNodeRestricted>false</executionNodeRestricted>
<name>Write error to log file -- ExecuteScript</name>
<relationships>
<autoTerminate>false</autoTerminate>
<name>failure</name>
</relationships>
<relationships>
<autoTerminate>false</autoTerminate>
<name>success</name>
</relationships>
<state>STOPPED</state>
<style/>
<type>org.apache.nifi.processors.script.ExecuteScript</type>
</processors>
</snippet>
<timestamp>02/14/2020 19:34:57 UTC</timestamp>
</template>

Don't have an account?
Coming from Hortonworks? Activate your account here