Support Questions
Find answers, ask questions, and share your expertise

jruby hbase scriipt stopped working after upgrade 5.16 to 6.3.4

Explorer

I have a jruby script for hbase, found on internet, I am not a jruby programer, but script was working but now stopped after upgrade from 5.16 to 6.3.4

 

TypeError: no public constructors for Java::OrgApacheHadoopHbaseClient::HBaseAdmin
<main> at list_small_regions.rb:59

 

# hbase org.jruby.Main list_small_regions.rb min_siz <namespace.tablename>
#
# Note: Please replace namespace.tablename with your namespace and table, eg NS1.MyTable. This value is case sensitive.

require 'digest'
require 'java'
java_import org.apache.hadoop.hbase.HBaseConfiguration
java_import org.apache.hadoop.hbase.client.HBaseAdmin
java_import org.apache.hadoop.hbase.TableName
java_import org.apache.hadoop.hbase.client.HTable
java_import org.apache.hadoop.hbase.HRegionInfo;
java_import org.apache.hadoop.hbase.client.Connection
java_import org.apache.hadoop.hbase.client.ConnectionFactory
java_import org.apache.hadoop.hbase.client.Table
java_import org.apache.hadoop.hbase.util.Bytes

def list_bigger_regions(table_name)
cluster_status = $admin.getClusterStatus()
master = cluster_status.getMaster()
biggers = []
cluster_status.getServers.each do |s|
cluster_status.getLoad(s).getRegionsLoad.each do |r|
# getRegionsLoad returns an array of arrays, where each array
# is 2 elements

# Filter out any regions that don't match the requested
# tablename
next unless r[1].get_name_as_string =~ /#{table_name}\,/
if r[1].getStorefileSizeMB() > $low_size
if r[1].get_name_as_string =~ /\.([^\.]+)\.$/
biggers.push $1
else
raise "Failed to get the encoded name for #{r[1].get_name_as_string}"
end
end
end
end
biggers
end

def regions_to_merge?(table_name)
bigger_regions = list_bigger_regions(table_name)
regions = $admin.getTableRegions(Bytes.toBytes(table_name));

filtered_regions = regions.reject do |r|
bigger_regions.include?(r.get_encoded_name)
end

puts "#{table_name},#{regions.length},#{bigger_regions.length},#{filtered_regions.length-1}"

filtered_regions.length
end

limit_batch = 1000
do_merge = false

config = HBaseConfiguration.create()
connection = ConnectionFactory.createConnection(config)
$admin = HBaseAdmin.new(connection)

# Handle command line parameters
$low_size = 1
if ARGV[0].to_i >= $low_size
$low_size=ARGV[0].to_i
end

puts "TABLE_NAME,TOT_REGIONS,REGIONS_OK,REGIONS_NEED_MERGED"

tables=$admin.listTables
tables.each do |table|
p "#{table}"
table_name = table.getNameAsString()
regions_to_merge?(table_name)
end
$admin.close

1 REPLY 1

Explorer

solution,

change 

#$admin = HBaseAdmin.new(connection)
$admin = connection.getAdmin();

 

leads to other changes

new script

 

#
# hbase org.jruby.Main list_small_regions.rb min_siz <namespace.tablename>
#
# Note: Please replace namespace.tablename with your namespace and table, eg NS1.MyTable. This value is case sensitive.

require 'digest'
require 'java'
java_import org.apache.hadoop.hbase.HBaseConfiguration
java_import org.apache.hadoop.hbase.client.HBaseAdmin
java_import org.apache.hadoop.hbase.TableName
java_import org.apache.hadoop.hbase.HRegionInfo;
java_import org.apache.hadoop.hbase.client.Connection
java_import org.apache.hadoop.hbase.client.ConnectionFactory
java_import org.apache.hadoop.hbase.client.Table
java_import org.apache.hadoop.hbase.util.Bytes

def list_bigger_regions(table_name)
cluster_status = $admin.getClusterStatus()
master = cluster_status.getMaster()
biggers = []
cluster_status.getServers.each do |s|
cluster_status.getLoad(s).getRegionsLoad.each do |r|
# getRegionsLoad returns an array of arrays, where each array
# is 2 elements

# Filter out any regions that don't match the requested
# tablename
next unless r[1].get_name_as_string =~ /#{table_name}\,/
if r[1].getStorefileSizeMB() > $low_size
if r[1].get_name_as_string =~ /\.([^\.]+)\.$/
biggers.push $1
else
raise "Failed to get the encoded name for #{r[1].get_name_as_string}"
end
end
end
end
biggers
end

def regions_to_merge?(table_name)
bigger_regions = list_bigger_regions(table_name)
#regions = $admin.getTableRegions(Bytes.toBytes(table_name));
regions = $admin.getTableRegions(table_name);

filtered_regions = regions.reject do |r|
bigger_regions.include?(r.get_encoded_name)
end

puts "#{table_name},#{regions.length},#{bigger_regions.length},#{filtered_regions.length-1}"

filtered_regions.length
end

limit_batch = 1000
do_merge = false

config = HBaseConfiguration.create()
connection = ConnectionFactory.createConnection(config)
#$admin = HBaseAdmin.new(connection)
$admin = connection.getAdmin();

# Handle command line parameters
$low_size = 1
if ARGV[0].to_i >= $low_size
$low_size=ARGV[0].to_i
end
all_tables=$admin.listTableNames
if ARGV.length > 1
p "ARGV - #{ARGV[1]}"
Table table = connection.getTable(TableName.valueOf("#{ARGV[1]}"));
tables.push table
else
tables=all_tables
end

puts "TABLE_NAME,TOT_REGIONS,REGIONS_OK,REGIONS_NEED_MERGED"

tables.each do |table|
#table_name = table.getName()
regions_to_merge?(table)
end
$admin.close

 

Take a Tour of the Community
Don't have an account?
Your experience may be limited. Sign in to explore more.