Created on 12-17-2020 10:53 AM - edited 09-16-2022 07:39 AM
I have a jruby script for hbase, found on internet, I am not a jruby programer, but script was working but now stopped after upgrade from 5.16 to 6.3.4
TypeError: no public constructors for Java::OrgApacheHadoopHbaseClient::HBaseAdmin
<main> at list_small_regions.rb:59
# hbase org.jruby.Main list_small_regions.rb min_siz <namespace.tablename>
#
# Note: Please replace namespace.tablename with your namespace and table, eg NS1.MyTable. This value is case sensitive.
require 'digest'
require 'java'
java_import org.apache.hadoop.hbase.HBaseConfiguration
java_import org.apache.hadoop.hbase.client.HBaseAdmin
java_import org.apache.hadoop.hbase.TableName
java_import org.apache.hadoop.hbase.client.HTable
java_import org.apache.hadoop.hbase.HRegionInfo;
java_import org.apache.hadoop.hbase.client.Connection
java_import org.apache.hadoop.hbase.client.ConnectionFactory
java_import org.apache.hadoop.hbase.client.Table
java_import org.apache.hadoop.hbase.util.Bytes
def list_bigger_regions(table_name)
cluster_status = $admin.getClusterStatus()
master = cluster_status.getMaster()
biggers = []
cluster_status.getServers.each do |s|
cluster_status.getLoad(s).getRegionsLoad.each do |r|
# getRegionsLoad returns an array of arrays, where each array
# is 2 elements
# Filter out any regions that don't match the requested
# tablename
next unless r[1].get_name_as_string =~ /#{table_name}\,/
if r[1].getStorefileSizeMB() > $low_size
if r[1].get_name_as_string =~ /\.([^\.]+)\.$/
biggers.push $1
else
raise "Failed to get the encoded name for #{r[1].get_name_as_string}"
end
end
end
end
biggers
end
def regions_to_merge?(table_name)
bigger_regions = list_bigger_regions(table_name)
regions = $admin.getTableRegions(Bytes.toBytes(table_name));
filtered_regions = regions.reject do |r|
bigger_regions.include?(r.get_encoded_name)
end
puts "#{table_name},#{regions.length},#{bigger_regions.length},#{filtered_regions.length-1}"
filtered_regions.length
end
limit_batch = 1000
do_merge = false
config = HBaseConfiguration.create()
connection = ConnectionFactory.createConnection(config)
$admin = HBaseAdmin.new(connection)
# Handle command line parameters
$low_size = 1
if ARGV[0].to_i >= $low_size
$low_size=ARGV[0].to_i
end
puts "TABLE_NAME,TOT_REGIONS,REGIONS_OK,REGIONS_NEED_MERGED"
tables=$admin.listTables
tables.each do |table|
p "#{table}"
table_name = table.getNameAsString()
regions_to_merge?(table_name)
end
$admin.close
Created 12-17-2020 12:48 PM
solution,
change
#$admin = HBaseAdmin.new(connection)
$admin = connection.getAdmin();
leads to other changes
new script
#
# hbase org.jruby.Main list_small_regions.rb min_siz <namespace.tablename>
#
# Note: Please replace namespace.tablename with your namespace and table, eg NS1.MyTable. This value is case sensitive.
require 'digest'
require 'java'
java_import org.apache.hadoop.hbase.HBaseConfiguration
java_import org.apache.hadoop.hbase.client.HBaseAdmin
java_import org.apache.hadoop.hbase.TableName
java_import org.apache.hadoop.hbase.HRegionInfo;
java_import org.apache.hadoop.hbase.client.Connection
java_import org.apache.hadoop.hbase.client.ConnectionFactory
java_import org.apache.hadoop.hbase.client.Table
java_import org.apache.hadoop.hbase.util.Bytes
def list_bigger_regions(table_name)
cluster_status = $admin.getClusterStatus()
master = cluster_status.getMaster()
biggers = []
cluster_status.getServers.each do |s|
cluster_status.getLoad(s).getRegionsLoad.each do |r|
# getRegionsLoad returns an array of arrays, where each array
# is 2 elements
# Filter out any regions that don't match the requested
# tablename
next unless r[1].get_name_as_string =~ /#{table_name}\,/
if r[1].getStorefileSizeMB() > $low_size
if r[1].get_name_as_string =~ /\.([^\.]+)\.$/
biggers.push $1
else
raise "Failed to get the encoded name for #{r[1].get_name_as_string}"
end
end
end
end
biggers
end
def regions_to_merge?(table_name)
bigger_regions = list_bigger_regions(table_name)
#regions = $admin.getTableRegions(Bytes.toBytes(table_name));
regions = $admin.getTableRegions(table_name);
filtered_regions = regions.reject do |r|
bigger_regions.include?(r.get_encoded_name)
end
puts "#{table_name},#{regions.length},#{bigger_regions.length},#{filtered_regions.length-1}"
filtered_regions.length
end
limit_batch = 1000
do_merge = false
config = HBaseConfiguration.create()
connection = ConnectionFactory.createConnection(config)
#$admin = HBaseAdmin.new(connection)
$admin = connection.getAdmin();
# Handle command line parameters
$low_size = 1
if ARGV[0].to_i >= $low_size
$low_size=ARGV[0].to_i
end
all_tables=$admin.listTableNames
if ARGV.length > 1
p "ARGV - #{ARGV[1]}"
Table table = connection.getTable(TableName.valueOf("#{ARGV[1]}"));
tables.push table
else
tables=all_tables
end
puts "TABLE_NAME,TOT_REGIONS,REGIONS_OK,REGIONS_NEED_MERGED"
tables.each do |table|
#table_name = table.getName()
regions_to_merge?(table)
end
$admin.close