[Fix][SqlTask] fix hive sqltask invalid connection parameters. (#4917)
* fix hive sqltask invalid connection parameters. * add setConnParams test method. * update base datasource test class code style. * add BaseDataSourceTest in the pom xml. * update hive conf list jdbc url stitching. * update hive datasource test class.json_split
zhuangchong
GitHub
parent
ccd8aaebab
commit
5bfe3fca67
|
|
@ -17,12 +17,16 @@
|
|||
|
||||
package org.apache.dolphinscheduler.dao.datasource;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
|
||||
import static org.apache.dolphinscheduler.common.Constants.USER;
|
||||
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.StringUtils;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Properties;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
|
@ -157,6 +161,11 @@ public abstract class BaseDataSource {
|
|||
separator = ":";
|
||||
break;
|
||||
case HIVE:
|
||||
if ("?".equals(otherParams.substring(0, 1))) {
|
||||
break;
|
||||
}
|
||||
separator = ";";
|
||||
break;
|
||||
case SPARK:
|
||||
case SQLSERVER:
|
||||
separator = ";";
|
||||
|
|
@ -178,6 +187,19 @@ public abstract class BaseDataSource {
|
|||
return DriverManager.getConnection(getJdbcUrl(), getUser(), getPassword());
|
||||
}
|
||||
|
||||
/**
|
||||
* the data source test connection
|
||||
* @param info Properties
|
||||
* @return Connection Connection
|
||||
* @throws Exception Exception
|
||||
*/
|
||||
public Connection getConnection(Properties info) throws Exception {
|
||||
Class.forName(driverClassSelector());
|
||||
info.setProperty(USER, getUser());
|
||||
info.setProperty(PASSWORD, getPassword());
|
||||
return DriverManager.getConnection(getJdbcUrl(), info);
|
||||
}
|
||||
|
||||
protected String filterOther(String otherParams) {
|
||||
return otherParams;
|
||||
}
|
||||
|
|
@ -226,6 +248,10 @@ public abstract class BaseDataSource {
|
|||
this.other = other;
|
||||
}
|
||||
|
||||
public void setConnParams(String connParams) {
|
||||
|
||||
}
|
||||
|
||||
public String getJavaSecurityKrb5Conf() {
|
||||
return javaSecurityKrb5Conf;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,13 +17,17 @@
|
|||
|
||||
package org.apache.dolphinscheduler.dao.datasource;
|
||||
|
||||
import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
|
||||
|
||||
import org.apache.dolphinscheduler.common.Constants;
|
||||
import org.apache.dolphinscheduler.common.enums.DbType;
|
||||
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.CommonUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
|
||||
import org.apache.dolphinscheduler.common.utils.StringUtils;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* data source of hive
|
||||
|
|
@ -100,4 +104,17 @@ public class HiveDataSource extends BaseDataSource {
|
|||
return super.getConnection();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setConnParams(String connParams) {
|
||||
// Verification parameters
|
||||
Map<String, String> connParamMap = CollectionUtils.stringToMap(connParams, SEMICOLON);
|
||||
if (connParamMap.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
StringBuilder otherSb = new StringBuilder();
|
||||
connParamMap.forEach((k, v) -> otherSb.append(String.format("%s=%s%s", k, v, SEMICOLON)));
|
||||
StringBuilder otherAppend = StringUtils.isNotBlank(getOther()) ? otherSb.append(getOther()) : otherSb.deleteCharAt(otherSb.length() - 1);
|
||||
super.setOther(otherAppend.toString());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,4 +158,38 @@ public class BaseDataSourceTest {
|
|||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSetConnParams() {
|
||||
|
||||
BaseDataSource hiveDataSource = new HiveDataSource();
|
||||
hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
|
||||
hiveDataSource.setDatabase("test");
|
||||
hiveDataSource.setPassword("123456");
|
||||
hiveDataSource.setUser("test");
|
||||
hiveDataSource.setConnParams("");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test", hiveDataSource.getJdbcUrl());
|
||||
|
||||
//set fake other
|
||||
hiveDataSource.setConnParams("hive.tez.container.size=20000;");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
|
||||
|
||||
hiveDataSource.setOther(null);
|
||||
hiveDataSource.setConnParams("hive.tez.container.size=20000");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
|
||||
|
||||
hiveDataSource.setOther(null);
|
||||
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
|
||||
|
||||
hiveDataSource.setOther("charset=UTF-8");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8", hiveDataSource.getJdbcUrl());
|
||||
|
||||
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000", hiveDataSource.getJdbcUrl());
|
||||
|
||||
hiveDataSource.setOther("charset=UTF-8;hive.exec.stagingdir=/tmp");
|
||||
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
|
||||
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000;hive.exec.stagingdir=/tmp", hiveDataSource.getJdbcUrl());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ public class HiveDataSourceTest {
|
|||
|
||||
hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
|
||||
Assert.assertEquals(
|
||||
"jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
|
||||
"jdbc:hive2://127.0.0.1:10000/test?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
|
||||
hiveDataSource.getJdbcUrl());
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -244,6 +244,9 @@ public class SqlTask extends AbstractTask {
|
|||
PreparedStatement stmt = null;
|
||||
ResultSet resultSet = null;
|
||||
try {
|
||||
|
||||
baseDataSource.setConnParams(sqlParameters.getConnParams());
|
||||
|
||||
// create connection
|
||||
connection = baseDataSource.getConnection();
|
||||
// create temp function
|
||||
|
|
|
|||
1
pom.xml
1
pom.xml
|
|
@ -990,6 +990,7 @@
|
|||
<include>**/dao/AlertDaoTest.java</include>
|
||||
<include>**/dao/datasource/OracleDataSourceTest.java</include>
|
||||
<include>**/dao/datasource/HiveDataSourceTest.java</include>
|
||||
<include>**/dao/datasource/BaseDataSourceTest.java</include>
|
||||
<include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>
|
||||
<include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>
|
||||
<include>**/dao/upgrade/UpgradeDaoTest.java</include>
|
||||
|
|
|
|||
Loading…
Reference in New Issue