From f0d21b7f90cdcce353ab6fc279b9cc376e46e536 Mon Sep 17 00:00:00 2001
From: Felix Cheung <felixcheung_m@hotmail.com>
Date: Thu, 8 Sep 2016 08:22:58 -0700
Subject: [PATCH] [SPARK-17442][SPARKR] Additional arguments in write.df are
not passed to data source
## What changes were proposed in this pull request?
additional options were not passed down in write.df.
## How was this patch tested?
unit tests
falaki shivaram
Author: Felix Cheung <felixcheung_m@hotmail.com>
Closes #15010 from felixcheung/testreadoptions.
---
R/pkg/R/DataFrame.R | 1 +
R/pkg/inst/tests/testthat/test_sparkSQL.R | 12 +++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index d7686972d2..40f1f0f442 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -2635,6 +2635,7 @@ setMethod("write.df",
write <- callJMethod(df@sdf, "write")
write <- callJMethod(write, "format", source)
write <- callJMethod(write, "mode", jmode)
+ write <- callJMethod(write, "options", options)
write <- callJMethod(write, "save", path)
})
diff --git a/R/pkg/inst/tests/testthat/test_sparkSQL.R b/R/pkg/inst/tests/testthat/test_sparkSQL.R
index a9bd325895..9d874a0988 100644
--- a/R/pkg/inst/tests/testthat/test_sparkSQL.R
+++ b/R/pkg/inst/tests/testthat/test_sparkSQL.R
@@ -208,7 +208,7 @@ test_that("create DataFrame from RDD", {
unsetHiveContext()
})
-test_that("read csv as DataFrame", {
+test_that("read/write csv as DataFrame", {
csvPath <- tempfile(pattern = "sparkr-test", fileext = ".csv")
mockLinesCsv <- c("year,make,model,comment,blank",
"\"2012\",\"Tesla\",\"S\",\"No comment\",",
@@ -243,7 +243,17 @@ test_that("read csv as DataFrame", {
expect_equal(count(withoutna2), 3)
expect_equal(count(where(withoutna2, withoutna2$make == "Dummy")), 0)
+ # writing csv file
+ csvPath2 <- tempfile(pattern = "csvtest2", fileext = ".csv")
+ write.df(df2, path = csvPath2, "csv", header = "true")
+ df3 <- read.df(csvPath2, "csv", header = "true")
+ expect_equal(nrow(df3), nrow(df2))
+ expect_equal(colnames(df3), colnames(df2))
+ csv <- read.csv(file = list.files(csvPath2, pattern = "^part", full.names = T)[[1]])
+ expect_equal(colnames(df3), colnames(csv))
+
unlink(csvPath)
+ unlink(csvPath2)
})
test_that("convert NAs to null type in DataFrames", {
--
GitLab