Ingesting EDI into HDFS using HDF 2.0

[#1]

1. Acquire an EDI File (GetFile, GetFTP, GetHTTP, GetSFTP, Fetch...)

2. Install open source nifi-edireader on NIFI 1.0.0

1. Download https://github.com/BerryWorksSoftware/edireader
2. Maven Install Berry Works EDIReader
3. Download https://github.com/mrcsparker/nifi-edireader-bundle
4. Maven packge nifi-edireader (must be Maven 3.3 or newer - may have to download and install separately from standard linux package)
5. cp nifi-edireader-nar/target/nifi-edireader-nar-0.0.1.nar to your NIFI/lib
6. Restart NiFi Service

3. Add EdiXML Processor and connect from EDI File input

4. Add extra processing, conversion or routing (TransformXML with XSLT or EvaluateXPATH) to convert to JSON

5. Land to HDFS (PutHDFS)

6. Used the Web Form linked below to generate a test EDI file.

ISA*00*          *00*          *ZZ*SENDER ID      *ZZ*RECEIVER ID    *010101*0101*U*00401*000000001*0*T*!
GS*IN*SENDER ID*APP RECEIVER*01010101*01010101*1*X*004010
ST*810*0001
BIG*20021208*00001**A999
N1*ST*Timothy Spann*9*122334455
N3*115 xxx ave
N4*xxxtown*nj*08520
N1*BT*Hortonworks*9*122334455
N3*5470 GREAT AMERICA PARKWAY
N4*santa clara*CA*95054
ITD*01*3*2**30**30*****60
FOB*PP
IT1**1*EA*200**UA*EAN
PID*F****Lamp
IT1**4*EA*50**UA*EAN
PID*F****Chair
TDS*2000
CAD*****Routing
ISS*30*CA
CTT*50
SE*19*0001
GE*1*1
IEA*1*000000001

7. Converted to XML

 <?xml version="1.0" encoding="UTF-8"?>
<ediroot>
   <interchange Standard="ANSI X.12"
                AuthorizationQual="00"
                Authorization="          "
                SecurityQual="00"
                Security="          "
                Date="010101"
                Time="0101"
                StandardsId="U"
                Version="00401"
                Control="000000001"
                AckRequest="0"
                TestIndicator="T">
      <sender>
         <address Id="SENDER ID      " Qual="ZZ"/>
      </sender>
      <receiver>
         <address Id="RECEIVER ID    " Qual="ZZ"/>
      </receiver>
      <group GroupType="IN"
             ApplSender="SENDER ID"
             ApplReceiver="APP RECEIVER"
             Date="01010101"
             Time="01010101"
             Control="1"
             StandardCode="X"
             StandardVersion="004010">
         <transaction DocType="810" Name="Invoice" Control="0001">
            <segment Id="BIG">
               <element Id="BIG01">20021208</element>
               <element Id="BIG02">00001</element>
               <element Id="BIG04">A999</element>
            </segment>
            <loop Id="N1">
               <segment Id="N1">
                  <element Id="N101">ST</element>
                  <element Id="N102">Timothy Spann</element>
                  <element Id="N103">9</element>
                  <element Id="N104">122334455</element>
               </segment>
               <segment Id="N3">
                  <element Id="N301">115 xxx ave</element>
               </segment>
               <segment Id="N4">
                  <element Id="N401">xxxstown</element>
                  <element Id="N402">nj</element>
                  <element Id="N403">08520</element>
               </segment>
            </loop>
            <loop Id="N1">
               <segment Id="N1">
                  <element Id="N101">BT</element>
                  <element Id="N102">Hortonworks</element>
                  <element Id="N103">9</element>
                  <element Id="N104">122334455</element>
               </segment>
               <segment Id="N3">
                  <element Id="N301">5470 GREAT AMERICA PARKWAY</element>
               </segment>
               <segment Id="N4">
                  <element Id="N401">santa clara</element>
                  <element Id="N402">CA</element>
                  <element Id="N403">95054</element>
               </segment>
            </loop>
            <segment Id="ITD">
               <element Id="ITD01">01</element>
               <element Id="ITD02">3</element>
               <element Id="ITD03">2</element>
               <element Id="ITD05">30</element>
               <element Id="ITD07">30</element>
               <element Id="ITD12">60</element>
            </segment>
            <segment Id="FOB">
               <element Id="FOB01">PP</element>
            </segment>
            <loop Id="IT1">
               <segment Id="IT1">
                  <element Id="IT102">1</element>
                  <element Id="IT103">EA</element>
                  <element Id="IT104">200</element>
                  <element Id="IT106">UA</element>
                  <element Id="IT107">EAN</element>
               </segment>
               <loop Id="PID">
                  <segment Id="PID">
                     <element Id="PID01">F</element>
                     <element Id="PID05">Lamp</element>
                  </segment>
               </loop>
            </loop>
            <loop Id="IT1">
               <segment Id="IT1">
                  <element Id="IT102">4</element>
                  <element Id="IT103">EA</element>
                  <element Id="IT104">50</element>
                  <element Id="IT106">UA</element>
                  <element Id="IT107">EAN</element>
               </segment>
               <loop Id="PID">
                  <segment Id="PID">
                     <element Id="PID01">F</element>
                     <element Id="PID05">Chair</element>
                  </segment>
               </loop>
            </loop>
            <segment Id="TDS">
               <element Id="TDS01">2000</element>
            </segment>
            <segment Id="CAD">
               <element Id="CAD05">Routing</element>
            </segment>
            <loop Id="ISS">
               <segment Id="ISS">
                  <element Id="ISS01">30</element>
                  <element Id="ISS02">CA</element>
               </segment>
            </loop>
            <segment Id="CTT">
               <element Id="CTT01">50</element>
            </segment>
         </transaction>
      </group>
   </interchange>
</ediroot>

Resources

• https://github.com/mrcsparker/nifi-edireader-bundle
• https://github.com/BerryWorksSoftware/edireader
• https://en.wikipedia.org/wiki/Electronic_data_interchange
• https://en.wikipedia.org/wiki/EDIFACT
• https://en.wikipedia.org/wiki/FORTRAS
• http://databene.org/edifatto.html
• https://sourceforge.net/projects/edifatto/
• https://secure.edidev.net/edidev-ca/samples/vbNetGen/WebFrmNetGen.aspx (Generate example EDI)