From 78a00fcf841284761e82dfd6ec47d5d732dec3f2 Mon Sep 17 00:00:00 2001 From: Mohammad Arshad Date: Mon, 30 Sep 2024 23:59:12 +0530 Subject: [PATCH] [Bug][Seatunnel-web][Hive] Filter system databases --- .../datasource/plugin/hive/HiveConfig.java | 15 ++-------- .../datasource/plugin/hive/HiveConstants.java | 5 ++-- .../plugin/hive/HiveDataSourceChannel.java | 29 +------------------ .../plugin/hive/HiveOptionRule.java | 3 -- .../plugin/hive/client/HiveClient.java | 24 +++++---------- 5 files changed, 13 insertions(+), 63 deletions(-) diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java index 7b5932119..355be853e 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConfig.java @@ -17,27 +17,16 @@ package org.apache.seatunnel.datasource.plugin.hive; -import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginInfo; import org.apache.seatunnel.datasource.plugin.api.DatasourcePluginTypeEnum; -import com.google.common.collect.Sets; - -import java.util.Set; - public class HiveConfig { - public static final String PLUGIN_NAME = "Hive"; public static final DataSourcePluginInfo HIVE_DATASOURCE_PLUGIN_INFO = DataSourcePluginInfo.builder() - .name(PLUGIN_NAME) - .icon(PLUGIN_NAME) + .name(HiveConstants.PLUGIN_NAME) + .icon(HiveConstants.PLUGIN_NAME) .version("1.0.0") .type(DatasourcePluginTypeEnum.DATABASE.getCode()) .build(); - - public static final Set HIVE_SYSTEM_DATABASES = Sets.newHashSet(); - - public static final OptionRule OPTION_RULE = - OptionRule.builder().required(HiveOptionRule.METASTORE_URI).build(); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java index 79d6fbf11..94eeca0af 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveConstants.java @@ -22,8 +22,7 @@ import java.util.Set; public class HiveConstants { - + public static final String PLUGIN_NAME = "Hive"; public static final Set HIVE_SYSTEM_DATABASES = - Sets.newHashSet( - "information_schema", "mysql", "performance_schema", "sys", "test", "hivedb"); + Sets.newHashSet("sys", "information_schema", "performance_schema", "mysql"); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java index f6066fa49..f5c49c5a7 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveDataSourceChannel.java @@ -28,9 +28,6 @@ import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,7 +61,7 @@ public List getTables( @Override public List getDatabases( @NonNull String pluginName, @NonNull Map requestParams) { - try (HiveClient hiveClient = HiveClient.createInstance(requestParams); ) { + try (HiveClient hiveClient = HiveClient.createInstance(requestParams)) { return hiveClient.getAllDatabases(); } } @@ -113,28 +110,4 @@ protected List getTableFields( return hiveClient.getFields(dbName, tableName); } } - - private static boolean checkHostConnectable(String host, int port) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 1000); - return true; - } catch (IOException e) { - return false; - } - } - - private boolean isNotSystemDatabase(String pluginName, String dbName) { - // FIXME,filters system databases - return true; - } - - private boolean convertToBoolean(Object value) { - if (value instanceof Boolean) { - return (Boolean) value; - } - if (value instanceof String) { - return value.equals("TRUE"); - } - return false; - } } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java index 776ca7f0b..e81f6cc31 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/HiveOptionRule.java @@ -91,7 +91,4 @@ public static OptionRule metadataRule() { // todo return OptionRule.builder().build(); } - - public static final Option TABLE_NAME = - Options.key("table").stringType().noDefaultValue().withDescription("hive table"); } diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java index 731d89148..bdc457baa 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-hive/src/main/java/org/apache/seatunnel/datasource/plugin/hive/client/HiveClient.java @@ -20,6 +20,7 @@ import org.apache.seatunnel.common.utils.ExceptionUtils; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; +import org.apache.seatunnel.datasource.plugin.hive.HiveConstants; import org.apache.seatunnel.datasource.plugin.hive.HiveOptionRule; import org.apache.commons.collections.CollectionUtils; @@ -40,6 +41,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkNotNull; @@ -94,17 +97,6 @@ public static HiveClient createInstance(Map reqParam) { } } - /*private static void authKerberos( - String kerberosKrb5ConfPath, String kerberosKeytabPath, String kerberosPrincipal) - throws IOException { - System.setProperty("java.security.krb5.conf", kerberosKrb5ConfPath); - Configuration configuration = new Configuration(); - configuration.set("hadoop.security.authentication", "Kerberos"); - configuration.setBoolean("hadoop.security.authorization", true); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation.loginUserFromKeytab(kerberosPrincipal, kerberosKeytabPath); - }*/ - public static void doKerberosAuthentication( Configuration configuration, String principal, String keytabPath) { if (StringUtils.isBlank(principal) || StringUtils.isBlank(keytabPath)) { @@ -142,17 +134,17 @@ public void close() { public List getAllDatabases() { try { - return hiveMetaStoreClient.getAllDatabases(); + List allDatabases = hiveMetaStoreClient.getAllDatabases(); + Set systemDatabases = HiveConstants.HIVE_SYSTEM_DATABASES; + return allDatabases.stream() + .filter(db -> !systemDatabases.contains(db)) + .collect(Collectors.toList()); } catch (Exception e) { log.error(ExceptionUtils.getMessage(e)); throw new DataSourcePluginException("get database names failed", e); } } - public List getAllTables(String dbName) { - return getAllTables(dbName, null, null); - } - public List getAllTables(String dbName, String filterName, Integer size) { try {