how to implement the hbase secure bulk load

后端 未结 1 1541
南旧
南旧 2021-01-29 00:26

I already created a bulk load in hbase in a kerberos cluster with a driver class similar to this (working):

public static void main(String[] args) {        
             


        
相关标签:
1条回答
  • 2021-01-29 00:49

    I will answer myself to my question :

    For this answer to work a table in hbase must already exist moreover HFile have to be already generated for the import

    import java.util.ArrayList;
    import java.util.List;
    import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.hbase.HBaseConfiguration;
    import org.apache.hadoop.hbase.HRegionInfo;
    import org.apache.hadoop.hbase.TableName;
    import org.apache.hadoop.hbase.client.Connection;
    import org.apache.hadoop.hbase.client.ConnectionFactory;
    import org.apache.hadoop.hbase.client.HBaseAdmin;
    import org.apache.hadoop.hbase.client.Table;
    import org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient;
    import org.apache.hadoop.hbase.security.UserProvider;
    import org.apache.hadoop.hbase.security.token.FsDelegationToken;
    import org.apache.hadoop.hbase.util.Pair;
    import org.apache.hadoop.security.UserGroupInformation;
    
    String keyTab = "pathtokeytabfile";
    String tableName = "tb_name";
    String pathToHFile = "/tmp/tmpfiles/";
    Configuration configuration = new Configuration();  
    
    configuration.set("hbase.zookeeper.quorum","ZK_QUORUM");
    configuration.set("hbase.zookeeper"+ ".property.clientPort","2181");
    configuration.set("hbase.master","MASTER:60000");
    configuration.set("hadoop.security.authentication", "Kerberos");
    configuration.set("hbase.security.authentication", "kerberos");
    
    
    //Obtaining kerberos authentication 
    
    UserGroupInformation.setConfiguration(configuration);
    
    UserGroupInformation.loginUserFromKeytab("here keytab", path to the key tab);
    
    HBaseAdmin.checkHBaseAvailable(configuration);
    
    System.out.println("HBase is running!");
    
    HBaseConfiguration.addHbaseResources(configuration);    
    
    Connection conn = ConnectionFactory.createConnection(configuration);
    
    Table table = conn.getTable(TableName.valueOf(tableName));
    
    HRegionInfo tbInfo = new HRegionInfo(table.getName());
    
    
    //path to the HFiles that need to be loaded 
    
    Path hfofDir = new Path(pathToHFile);
    
    //acquiring user token for authentication 
    
    UserProvider up = UserProvider.instantiate(configuration);
    
    FsDelegationToken fsDelegationToken = new FsDelegationToken(up, "name of the key tab user");
    
    fsDelegationToken.acquireDelegationToken(hfofDir.getFileSystem(configuration));
    
    //preparing  for the bulk load
    
    SecureBulkLoadClient secureBulkLoadClient = new SecureBulkLoadClient(table);
    
    String bulkToken = secureBulkLoadClient.prepareBulkLoad(table.getName());
    
    System.out.println(bulkToken);
    
    //creating the family list (list of family names and path to the hfile corresponding to the family name)
    
    final List<Pair<byte[], String>> famPaths = new ArrayList<>();
    
    Pair p = new Pair();
    
    //name of the family 
    p.setFirst("nameofthefamily".getBytes());
    
    //path to the HFile (HFile are organized in folder with the name of the family)
    p.setSecond("/tmp/tmpfiles/INTRO/nameofthefilehere");
    
    famPaths.add(p);
    
    //bulk loading ,using the secure bulk load client
    
    secureBulkLoadClient.bulkLoadHFiles(famPaths, fsDelegationToken.getUserToken(), bulkToken, tbInfo.getStartKey());
    
    System.out.println("Bulk Load Completed..");    
    
    0 讨论(0)
提交回复
热议问题