Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

how can i get the full extended tweet using GetTwitter

Highlighted

how can i get the full extended tweet using GetTwitter

New Contributor

hello,

I am new to nifi, i have a problem getting the full text of the tweets,

 

let me explain,

 

When i use evaluateJasonPath processor to get the tweet text using the json path: $.text it only gets the 140 character and not the whole tweet

1.jpg

for example, in this part

 

{
  "created_at" : "Thu Mar 26 12:59:13 +0000 2020",
  "id" : 1243160580980170754,
  "id_str" : "1243160580980170754",
  "text" : "RT @queenmaine__: Y’all make sure you call your Student Loan Servicer and take advantage of this Covid-19 Forbearance. It’s non capitalized…",
  "source" : "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
  "truncated" : false,
  "in_reply_to_status_id" : null,
  "in_reply_to_status_id_str" : null,
  "in_reply_to_user_id" : null,
  "in_reply_to_user_id_str" : null,
  "in_reply_to_screen_name" : null,
}

 $.text in evaluateJsonPath returns this

"RT @queenmaine__: Y’all make sure you call your Student Loan Servicer and take advantage of this Covid-19 Forbearance. It’s non capitalized…"

but this is not the whole tweet text  the "..." in the tweet indicates that there are more chacters in the tweet.

 

then i read that for more than 140 character there's another section in json called extended_tweet but it is not always there, sometimes its under the retweet_status like this example.

 

{
  "created_at" : "Thu Mar 26 12:59:13 +0000 2020",
  "id" : 1243160580980170754,
  "id_str" : "1243160580980170754",
  "text" : "RT @queenmaine__: Y’all make sure you call your Student Loan Servicer and take advantage of this Covid-19 Forbearance. It’s non capitalized…",
  "source" : "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
  "truncated" : false,
  "in_reply_to_status_id" : null,
  "in_reply_to_status_id_str" : null,
  "in_reply_to_user_id" : null,
  "in_reply_to_user_id_str" : null,
  "in_reply_to_screen_name" : null,
  "user" : {
    "id" : 750224880625283072,
    "id_str" : "750224880625283072",
    "name" : "Leni Chanel Quick",
    "screen_name" : "LeniChanel",
    "location" : null,
    "url" : null,
    "description" : "Writer, editor, educator, photographer - servant to all; constant conduit.",
    "translator_type" : "none",
    "protected" : false,
    "verified" : false,
    "followers_count" : 124,
    "friends_count" : 899,
    "listed_count" : 1,
    "favourites_count" : 44025,
    "statuses_count" : 12667,
    "created_at" : "Tue Jul 05 07:08:45 +0000 2016",
    "utc_offset" : null,
    "time_zone" : null,
    "geo_enabled" : false,
    "lang" : null,
    "contributors_enabled" : false,
    "is_translator" : false,
    "profile_background_color" : "000000",
    "profile_background_image_url" : "http://abs.twimg.com/images/themes/theme1/bg.png",
    "profile_background_image_url_https" : "https://abs.twimg.com/images/themes/theme1/bg.png",
    "profile_background_tile" : false,
    "profile_link_color" : "F58EA8",
    "profile_sidebar_border_color" : "000000",
    "profile_sidebar_fill_color" : "000000",
    "profile_text_color" : "000000",
    "profile_use_background_image" : false,
    "profile_image_url" : "http://pbs.twimg.com/profile_images/778542255548223489/MqlqA8Ny_normal.jpg",
    "profile_image_url_https" : "https://pbs.twimg.com/profile_images/778542255548223489/MqlqA8Ny_normal.jpg",
    "default_profile" : false,
    "default_profile_image" : false,
    "following" : null,
    "follow_request_sent" : null,
    "notifications" : null
  },
  "geo" : null,
  "coordinates" : null,
  "place" : null,
  "contributors" : null,
  "retweeted_status" : {
    "created_at" : "Wed Mar 25 18:01:55 +0000 2020",
    "id" : 1242874369745002498,
    "id_str" : "1242874369745002498",
    "text" : "Y’all make sure you call your Student Loan Servicer and take advantage of this Covid-19 Forbearance. It’s non capit… https://t.co/UyrymxHHRM",
    "source" : "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>",
    "truncated" : true,
    "in_reply_to_status_id" : null,
    "in_reply_to_status_id_str" : null,
    "in_reply_to_user_id" : null,
    "in_reply_to_user_id_str" : null,
    "in_reply_to_screen_name" : null,
    "user" : {
      "id" : 2924888576,
      "id_str" : "2924888576",
      "name" : "cuh-ween",
      "screen_name" : "queenmaine__",
      "location" : "Jer 29:11",
      "url" : null,
      "description" : "be love , spread love  20somethings  authentic. FMT",
      "translator_type" : "none",
      "protected" : false,
      "verified" : false,
      "followers_count" : 1074,
      "friends_count" : 690,
      "listed_count" : 5,
      "favourites_count" : 33790,
      "statuses_count" : 17910,
      "created_at" : "Wed Dec 10 00:40:52 +0000 2014",
      "utc_offset" : null,
      "time_zone" : null,
      "geo_enabled" : true,
      "lang" : null,
      "contributors_enabled" : false,
      "is_translator" : false,
      "profile_background_color" : "C0DEED",
      "profile_background_image_url" : "http://abs.twimg.com/images/themes/theme1/bg.png",
      "profile_background_image_url_https" : "https://abs.twimg.com/images/themes/theme1/bg.png",
      "profile_background_tile" : false,
      "profile_link_color" : "1DA1F2",
      "profile_sidebar_border_color" : "C0DEED",
      "profile_sidebar_fill_color" : "DDEEF6",
      "profile_text_color" : "333333",
      "profile_use_background_image" : true,
      "profile_image_url" : "http://pbs.twimg.com/profile_images/1210265891553841152/9HN8vhWw_normal.jpg",
      "profile_image_url_https" : "https://pbs.twimg.com/profile_images/1210265891553841152/9HN8vhWw_normal.jpg",
      "profile_banner_url" : "https://pbs.twimg.com/profile_banners/2924888576/1585009600",
      "default_profile" : true,
      "default_profile_image" : false,
      "following" : null,
      "follow_request_sent" : null,
      "notifications" : null
    },
    "geo" : null,
    "coordinates" : null,
    "place" : null,
    "contributors" : null,
    "is_quote_status" : false,
    "extended_tweet" : {
      "full_text" : "Y’all make sure you call your Student Loan Servicer and take advantage of this Covid-19 Forbearance. It’s non capitalized and it HAS TO BE REQUESTED. 0% interest effective 3/13/2020. Don’t let them play y’all. You’re welcome \n\n-Your Fav Student Loan Counselor",
      "display_text_range" : [ 0, 259 ],
      "entities" : {
        "hashtags" : [ ],
        "urls" : [ ],
        "user_mentions" : [ ],
        "symbols" : [ ]
      }
    },
    "quote_count" : 107,
    "reply_count" : 76,
    "retweet_count" : 7761,
    "favorite_count" : 15693,
    "entities" : {
      "hashtags" : [ ],
      "urls" : [ {
        "url" : "https://t.co/UyrymxHHRM",
        "expanded_url" : "https://twitter.com/i/web/status/1242874369745002498",
        "display_url" : "twitter.com/i/web/status/1…",
        "indices" : [ 117, 140 ]
      } ],
      "user_mentions" : [ ],
      "symbols" : [ ]
    },
    "favorited" : false,
    "retweeted" : false,
    "filter_level" : "low",
    "lang" : "en"
  },
  "is_quote_status" : false,
  "quote_count" : 0,
  "reply_count" : 0,
  "retweet_count" : 0,
  "favorite_count" : 0,
  "entities" : {
    "hashtags" : [ ],
    "urls" : [ ],
    "user_mentions" : [ {
      "screen_name" : "queenmaine__",
      "name" : "cuh-ween",
      "id" : 2924888576,
      "id_str" : "2924888576",
      "indices" : [ 3, 16 ]
    } ],
    "symbols" : [ ]
  },
  "favorited" : false,
  "retweeted" : false,
  "filter_level" : "low",
  "lang" : "en",
  "timestamp_ms" : "1585227553969"
}

 and sometimes its under the root itself ($.extended_tweet) like this example

{
  "created_at" : "Thu Mar 26 12:00:44 +0000 2020",
  "id" : 1243145860537626631,
  "id_str" : "1243145860537626631",
  "text" : "If COVID-19 caused your NYC small business sales to drop 25% or more, you may be eligible for zero interest loans o… https://t.co/Kg6Q2xQ5vC",
  "source" : "<a href=\"https://www.hootsuite.com\" rel=\"nofollow\">Hootsuite Inc.</a>",
  "truncated" : true,
  "in_reply_to_status_id" : null,
  "in_reply_to_status_id_str" : null,
  "in_reply_to_user_id" : null,
  "in_reply_to_user_id_str" : null,
  "in_reply_to_screen_name" : null,
  "user" : {
    "id" : 24234143,
    "id_str" : "24234143",
    "name" : "The Lang Entrepreneurship Center",
    "screen_name" : "LangCenter_CBS",
    "location" : "New York, NY",
    "url" : "http://linktr.ee/langcentercbs",
    "description" : "Columbia Business School’s Lang Entrepreneurship Center is at the heart of the entrepreneurial, innovation, & VC communities for both students & alumni.",
    "translator_type" : "none",
    "protected" : false,
    "verified" : false,
    "followers_count" : 6175,
    "friends_count" : 2075,
    "listed_count" : 297,
    "favourites_count" : 1946,
    "statuses_count" : 6221,
    "created_at" : "Fri Mar 13 19:28:56 +0000 2009",
    "utc_offset" : null,
    "time_zone" : null,
    "geo_enabled" : true,
    "lang" : null,
    "contributors_enabled" : false,
    "is_translator" : false,
    "profile_background_color" : "666666",
    "profile_background_image_url" : "http://abs.twimg.com/images/themes/theme1/bg.png",
    "profile_background_image_url_https" : "https://abs.twimg.com/images/themes/theme1/bg.png",
    "profile_background_tile" : false,
    "profile_link_color" : "0081CC",
    "profile_sidebar_border_color" : "E0E0E0",
    "profile_sidebar_fill_color" : "E0E0E0",
    "profile_text_color" : "464646",
    "profile_use_background_image" : false,
    "profile_image_url" : "http://pbs.twimg.com/profile_images/887030611038408704/feA93Ic7_normal.jpg",
    "profile_image_url_https" : "https://pbs.twimg.com/profile_images/887030611038408704/feA93Ic7_normal.jpg",
    "profile_banner_url" : "https://pbs.twimg.com/profile_banners/24234143/1499269315",
    "default_profile" : false,
    "default_profile_image" : false,
    "following" : null,
    "follow_request_sent" : null,
    "notifications" : null
  },
  "geo" : null,
  "coordinates" : null,
  "place" : null,
  "contributors" : null,
  "is_quote_status" : false,
  "extended_tweet" : {
    "full_text" : "If COVID-19 caused your NYC small business sales to drop 25% or more, you may be eligible for zero interest loans of up to $75,000. https://t.co/Hw6Vn9moXI",
    "display_text_range" : [ 0, 155 ],
    "entities" : {
      "hashtags" : [ ],
      "urls" : [ {
        "url" : "https://t.co/Hw6Vn9moXI",
        "expanded_url" : "https://on.nyc.gov/2JaD0vd",
        "display_url" : "on.nyc.gov/2JaD0vd",
        "indices" : [ 132, 155 ]
      } ],
      "user_mentions" : [ ],
      "symbols" : [ ]
    }
  },
  "quote_count" : 0,
  "reply_count" : 0,
  "retweet_count" : 0,
  "favorite_count" : 0,
  "entities" : {
    "hashtags" : [ ],
    "urls" : [ {
      "url" : "https://t.co/Kg6Q2xQ5vC",
      "expanded_url" : "https://twitter.com/i/web/status/1243145860537626631",
      "display_url" : "twitter.com/i/web/status/1…",
      "indices" : [ 117, 140 ]
    } ],
    "user_mentions" : [ ],
    "symbols" : [ ]
  },
  "favorited" : false,
  "retweeted" : false,
  "possibly_sensitive" : false,
  "filter_level" : "low",
  "lang" : "en",
  "timestamp_ms" : "1585224044342"
}

and sometimes it is not there at all.

 

my problem is that i cant find a json path to use in the evaluate json path processor to get the whole tweet text. Is there is any solution for the problem ??

 

thanks in advance and have a nice day.

Don't have an account?
Coming from Hortonworks? Activate your account here