Build and run KafkaSpout on secure cluster

[#1]

ENV: HDP-2.5

Java : openjdk version "1.8.0_111"

the following storm topology consist of a KafkaSpout and a SinkTypeBolt

Step 1: Create pom.xml with following dependencies

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>

   <groupId>hadoop</groupId>
   <artifactId>KafkaSpoutStorm</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <packaging>jar</packaging>

   <name>stormkafka</name>
   <url>http://maven.apache.org</url>

   <properties>
      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
   </properties>

   <repositories>
      <repository>
         <id>HDPReleases</id>
         <name>HDP Releases</name>
         <url>http://repo.hortonworks.com/content/repositories/public</url>
         <layout>default</layout>
      </repository>
      <repository>
         <id>HDPJetty</id>
         <name>Hadoop Jetty</name>
         <url>http://repo.hortonworks.com/content/repositories/jetty-hadoop/</url>
         <layout>default</layout>
      </repository>
   </repositories>


   <dependencies>
      <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
         <version>4.11</version>
         <scope>test</scope>
      </dependency>
      <dependency>
         <groupId>org.apache.storm</groupId>
         <artifactId>storm-core</artifactId>
         <version>1.0.1.2.5.3.0-37</version>
         <scope>provided</scope>
         <exclusions>
            <exclusion>
               <groupId>org.slf4j</groupId>
               <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
         </exclusions>
      </dependency>
      <dependency>
         <groupId>org.apache.kafka</groupId>
         <artifactId>kafka_2.10</artifactId>
         <version>0.10.0.2.5.3.0-37</version>
         <exclusions>
            <exclusion>
               <groupId>org.slf4j</groupId>
               <artifactId>slf4j-log4j12</artifactId>
            </exclusion>
         </exclusions>
      </dependency>
      <dependency>
         <groupId>org.apache.storm</groupId>
         <artifactId>storm-kafka</artifactId>
         <version>1.0.1.2.5.3.0-37</version>
      </dependency>
      <dependency>
         <groupId>org.apache.storm</groupId>
         <artifactId>storm-hdfs</artifactId>
         <version>1.0.1.2.5.3.0-37</version>
      </dependency>
      <dependency>
         <groupId>com.googlecode.json-simple</groupId>
         <artifactId>json-simple</artifactId>
         <version>1.1</version>
      </dependency>
   </dependencies>

   <build>
      <plugins>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>1.4</version>
            <configuration>
               <createDependencyReducedPom>true</createDependencyReducedPom>
            </configuration>
            <executions>
               <execution>
                  <phase>package</phase>
                  <goals>
                     <goal>shade</goal>
                  </goals>
                  <configuration>
                     <transformers>
                        <transformer
                           implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
                        <transformer
                           implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                           <mainClass>com.rajkrrsingh.storm.Topology</mainClass>
                        </transformer>
                     </transformers>
                     <filters>
                        <filter>
                           <artifact>*:*</artifact>
                           <excludes>
                              <exclude>META-INF/*.SF</exclude>
                              <exclude>META-INF/*.DSA</exclude>
                              <exclude>META-INF/*.RSA</exclude>
                           </excludes>
                        </filter>
                     </filters>
                  </configuration>
               </execution>
            </executions>
         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-resources-plugin</artifactId>
            <version>2.4</version>
         </plugin>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-source-plugin</artifactId>
            <executions>
               <execution>
                  <id>attach-sources</id>
                  <goals>
                     <goal>jar</goal>
                  </goals>
               </execution>
            </executions>
         </plugin>
      </plugins>
      <resources>
         <resource>
            <directory>src/main/java</directory>
            <includes>
               <include> **/*.properties</include>
            </includes>
         </resource>
      </resources>
   </build>

</project>

Step 2: clone the git repo to get the complete code

git clone https://github.com/rajkrrsingh/KafkaSpoutStorm.git

Step 3: modify default_config.properties according to your cluster

Step 4: build using maven, this will create a fat jar in target folder

mvn clean package

Step 5: Now Run it on storm cluster

storm jar KafkaSpoutStorm-0.0.1-SNAPSHOT.jar com.rajkrrsingh.storm.Topology