Ingesting EDI into HDFS using HDF 2.0

[#1]

1. Acquire an EDI File (GetFile, GetFTP, GetHTTP, GetSFTP, Fetch...)

2. Install open source nifi-edireader on NIFI 1.0.0

1. Download https://github.com/BerryWorksSoftware/edireader
2. Maven Install Berry Works EDIReader
3. Download https://github.com/mrcsparker/nifi-edireader-bundle
4. Maven packge nifi-edireader (must be Maven 3.3 or newer - may have to download and install separately from standard linux package)
5. cp nifi-edireader-nar/target/nifi-edireader-nar-0.0.1.nar to your NIFI/lib
6. Restart NiFi Service

3. Add EdiXML Processor and connect from EDI File input

8293-edixml.png

4. Add extra processing, conversion or routing (TransformXML with XSLT or EvaluateXPATH) to convert to JSON

5. Land to HDFS (PutHDFS)

8292-edi1.png

6. Used the Web Form linked below to generate a test EDI file.

ISA*00*          *00*          *ZZ*SENDER ID      *ZZ*RECEIVER ID    *010101*0101*U*00401*000000001*0*T*!
GS*IN*SENDER ID*APP RECEIVER*01010101*01010101*1*X*004010
ST*810*0001
BIG*20021208*00001**A999
N1*ST*Timothy Spann*9*122334455
N3*115 xxx ave
N4*xxxtown*nj*08520
N1*BT*Hortonworks*9*122334455
N3*5470 GREAT AMERICA PARKWAY
N4*santa clara*CA*95054
ITD*01*3*2**30**30*****60
FOB*PP
IT1**1*EA*200**UA*EAN
PID*F****Lamp
IT1**4*EA*50**UA*EAN
PID*F****Chair
TDS*2000
CAD*****Routing
ISS*30*CA
CTT*50
SE*19*0001
GE*1*1
IEA*1*000000001

7. Converted to XML

 <?xml version="1.0" encoding="UTF-8"?>
<ediroot>
   <interchange Standard="ANSI X.12"
                AuthorizationQual="00"
                Authorization="          "
                SecurityQual="00"
                Security="          "
                Date="010101"
                Time="0101"
                StandardsId="U"
                Version="00401"
                Control="000000001"
                AckRequest="0"
                TestIndicator="T">
      <sender>
         <address Id="SENDER ID      " Qual="ZZ"/>
      </sender>
      <receiver>
         <address Id="RECEIVER ID    " Qual="ZZ"/>
      </receiver>
      <group GroupType="IN"
             ApplSender="SENDER ID"
             ApplReceiver="APP RECEIVER"
             Date="01010101"
             Time="01010101"
             Control="1"
             StandardCode="X"
             StandardVersion="004010">
         <transaction DocType="810" Name="Invoice" Control="0001">
            <segment Id="BIG">
               <element Id="BIG01">20021208</element>
               <element Id="BIG02">00001</element>
               <element Id="BIG04">A999</element>
            </segment>
            <loop Id="N1">
               <segment Id="N1">
                  <element Id="N101">ST</element>
                  <element Id="N102">Timothy Spann</element>
                  <element Id="N103">9</element>
                  <element Id="N104">122334455</element>
               </segment>
               <segment Id="N3">
                  <element Id="N301">115 xxx ave</element>
               </segment>
               <segment Id="N4">
                  <element Id="N401">xxxstown</element>
                  <element Id="N402">nj</element>
                  <element Id="N403">08520</element>
               </segment>
            </loop>
            <loop Id="N1">
               <segment Id="N1">
                  <element Id="N101">BT</element>
                  <element Id="N102">Hortonworks</element>
                  <element Id="N103">9</element>
                  <element Id="N104">122334455</element>
               </segment>
               <segment Id="N3">
                  <element Id="N301">5470 GREAT AMERICA PARKWAY</element>
               </segment>
               <segment Id="N4">
                  <element Id="N401">santa clara</element>
                  <element Id="N402">CA</element>
                  <element Id="N403">95054</element>
               </segment>
            </loop>
            <segment Id="ITD">
               <element Id="ITD01">01</element>
               <element Id="ITD02">3</element>
               <element Id="ITD03">2</element>
               <element Id="ITD05">30</element>
               <element Id="ITD07">30</element>
               <element Id="ITD12">60</element>
            </segment>
            <segment Id="FOB">
               <element Id="FOB01">PP</element>
            </segment>
            <loop Id="IT1">
               <segment Id="IT1">
                  <element Id="IT102">1</element>
                  <element Id="IT103">EA</element>
                  <element Id="IT104">200</element>
                  <element Id="IT106">UA</element>
                  <element Id="IT107">EAN</element>
               </segment>
               <loop Id="PID">
                  <segment Id="PID">
                     <element Id="PID01">F</element>
                     <element Id="PID05">Lamp</element>
                  </segment>
               </loop>
            </loop>
            <loop Id="IT1">
               <segment Id="IT1">
                  <element Id="IT102">4</element>
                  <element Id="IT103">EA</element>
                  <element Id="IT104">50</element>
                  <element Id="IT106">UA</element>
                  <element Id="IT107">EAN</element>
               </segment>
               <loop Id="PID">
                  <segment Id="PID">
                     <element Id="PID01">F</element>
                     <element Id="PID05">Chair</element>
                  </segment>
               </loop>
            </loop>
            <segment Id="TDS">
               <element Id="TDS01">2000</element>
            </segment>
            <segment Id="CAD">
               <element Id="CAD05">Routing</element>
            </segment>
            <loop Id="ISS">
               <segment Id="ISS">
                  <element Id="ISS01">30</element>
                  <element Id="ISS02">CA</element>
               </segment>
            </loop>
            <segment Id="CTT">
               <element Id="CTT01">50</element>
            </segment>
         </transaction>
      </group>
   </interchange>
</ediroot>

Resources

• https://github.com/mrcsparker/nifi-edireader-bundle
• https://github.com/BerryWorksSoftware/edireader
• https://en.wikipedia.org/wiki/Electronic_data_interchange
• https://en.wikipedia.org/wiki/EDIFACT
• https://en.wikipedia.org/wiki/FORTRAS
• http://databene.org/edifatto.html
• https://sourceforge.net/projects/edifatto/
• https://secure.edidev.net/edidev-ca/samples/vbNetGen/WebFrmNetGen.aspx (Generate example EDI)