Skip to content
Snippets Groups Projects
Commit 2b7ab814 authored by William Benton's avatar William Benton Committed by Michael Armbrust
Browse files

[SPARK-3329][SQL] Don't depend on Hive SET pair ordering in tests.

This fixes some possible spurious test failures in `HiveQuerySuite` by comparing sets of key-value pairs as sets, rather than as lists.

Author: William Benton <willb@redhat.com>
Author: Aaron Davidson <aaron@databricks.com>

Closes #2220 from willb/spark-3329 and squashes the following commits:

3b3e205 [William Benton] Collapse collectResults case match in HiveQuerySuite
6525d8e [William Benton] Handle cases where SET returns Rows of (single) strings
cf11b0e [Aaron Davidson] Fix flakey HiveQuerySuite test
parent dc1dbf20
No related branches found
No related tags found
No related merge requests found
...@@ -558,62 +558,67 @@ class HiveQuerySuite extends HiveComparisonTest { ...@@ -558,62 +558,67 @@ class HiveQuerySuite extends HiveComparisonTest {
val testKey = "spark.sql.key.usedfortestonly" val testKey = "spark.sql.key.usedfortestonly"
val testVal = "test.val.0" val testVal = "test.val.0"
val nonexistentKey = "nonexistent" val nonexistentKey = "nonexistent"
val KV = "([^=]+)=([^=]*)".r
def collectResults(rdd: SchemaRDD): Set[(String, String)] =
rdd.collect().map {
case Row(key: String, value: String) => key -> value
case Row(KV(key, value)) => key -> value
}.toSet
clear() clear()
// "set" itself returns all config variables currently specified in SQLConf. // "set" itself returns all config variables currently specified in SQLConf.
// TODO: Should we be listing the default here always? probably... // TODO: Should we be listing the default here always? probably...
assert(sql("SET").collect().size == 0) assert(sql("SET").collect().size == 0)
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0)) collectResults(hql(s"SET $testKey=$testVal"))
} }
assert(hiveconf.get(testKey, "") == testVal) assert(hiveconf.get(testKey, "") == testVal)
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0)) collectResults(hql("SET"))
} }
sql(s"SET ${testKey + testKey}=${testVal + testVal}") sql(s"SET ${testKey + testKey}=${testVal + testVal}")
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal) assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) { assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
sql(s"SET").collect().map(_.getString(0)) collectResults(hql("SET"))
} }
// "set key" // "set key"
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql(s"SET $testKey").collect().map(_.getString(0)) collectResults(hql(s"SET $testKey"))
} }
assertResult(Array(s"$nonexistentKey=<undefined>")) { assertResult(Set(nonexistentKey -> "<undefined>")) {
sql(s"SET $nonexistentKey").collect().map(_.getString(0)) collectResults(hql(s"SET $nonexistentKey"))
} }
// Assert that sql() should have the same effects as sql() by repeating the above using sql(). // Assert that sql() should have the same effects as sql() by repeating the above using sql().
clear() clear()
assert(sql("SET").collect().size == 0) assert(sql("SET").collect().size == 0)
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql(s"SET $testKey=$testVal").collect().map(_.getString(0)) collectResults(sql(s"SET $testKey=$testVal"))
} }
assert(hiveconf.get(testKey, "") == testVal) assert(hiveconf.get(testKey, "") == testVal)
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql("SET").collect().map(_.getString(0)) collectResults(sql("SET"))
} }
sql(s"SET ${testKey + testKey}=${testVal + testVal}") sql(s"SET ${testKey + testKey}=${testVal + testVal}")
assert(hiveconf.get(testKey + testKey, "") == testVal + testVal) assert(hiveconf.get(testKey + testKey, "") == testVal + testVal)
assertResult(Array(s"$testKey=$testVal", s"${testKey + testKey}=${testVal + testVal}")) { assertResult(Set(testKey -> testVal, (testKey + testKey) -> (testVal + testVal))) {
sql("SET").collect().map(_.getString(0)) collectResults(sql("SET"))
} }
assertResult(Array(s"$testKey=$testVal")) { assertResult(Set(testKey -> testVal)) {
sql(s"SET $testKey").collect().map(_.getString(0)) collectResults(sql(s"SET $testKey"))
} }
assertResult(Array(s"$nonexistentKey=<undefined>")) { assertResult(Set(nonexistentKey -> "<undefined>")) {
sql(s"SET $nonexistentKey").collect().map(_.getString(0)) collectResults(sql(s"SET $nonexistentKey"))
} }
clear() clear()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment