Merge pull request #229 from r-transit/dev/NA_strings_and_field_types

Empty strings as NA and numeric field types

Author: polettif

R/convert_types.R

@@ -22,17 +22,31 @@ convert_types <- function(gtfs_list, conversion_table, conversion_function) {
 }
 
 convert_char_to_date <- function(gtfs_list) {
-  convert_types(gtfs_list, gtfs_reference_types$Date, .parse_gtfsio_date)
+  convert_types(gtfs_list, gtfs_reference_types[["Date"]], .parse_gtfsio_date)
 }
 
 convert_date_to_char <- function(gtfs_obj) {
-  convert_types(gtfs_obj, gtfs_reference_types$Date, .date_as_gtfsio_char)
+  convert_types(gtfs_obj, gtfs_reference_types[["Date"]], .date_as_gtfsio_char)
 }
 
 convert_char_to_hms <- function(gtfs_list) {
-  convert_types(gtfs_list, gtfs_reference_types$Time, hhmmss_to_hms)
+  convert_types(gtfs_list, gtfs_reference_types[["Time"]], hhmmss_to_hms)
 }
 
 convert_hms_to_char <- function(gtfs_obj) {
-  convert_types(gtfs_obj, gtfs_reference_types$Time, hms_to_hhmmss)
+  convert_types(gtfs_obj, gtfs_reference_types[["Time"]], hms_to_hhmmss)
+}
+
+# reading only
+convert_char_to_int <- function(gtfs_obj) {
+  int_ref = do.call("rbind", 
+                    gtfs_reference_types[c("Integer", "Non-negative integer", "Non-null integer",
+                                           "Non-zero integer", "Positive integer")])
+  convert_types(gtfs_obj, int_ref, as.integer)
+}
+
+convert_char_to_num <- function(gtfs_obj) {
+  num_ref = do.call("rbind", 
+                    gtfs_reference_types[c("Float", "Positive float", "Non-negative float")])
+  convert_types(gtfs_obj, num_ref, as.double)
 }

R/tidygtfs.R

@@ -59,6 +59,9 @@ gtfs_to_tidygtfs = function(gtfs_list, files = NULL) {
   x <- convert_list_tables_to_tibbles(x)
   x <- convert_list_json_to_sf(x)
 
+  # convert empty strings "" to NA
+  x <- empty_strings_to_na(x)
+  
   # gtfs class base structure
   x <- gtfsio::new_gtfs(x)
   class(x) <- c("tidygtfs", "gtfs", "list")
@@ -70,6 +73,8 @@ gtfs_to_tidygtfs = function(gtfs_list, files = NULL) {
 prepare_tidygtfs_fields = function(gtfs_obj) {
   gtfs_obj <- convert_char_to_hms(gtfs_obj)
   gtfs_obj <- convert_char_to_date(gtfs_obj)
+  gtfs_obj <- convert_char_to_int(gtfs_obj)
+  gtfs_obj <- convert_char_to_num(gtfs_obj)
   return(gtfs_obj)
 }
 

R/utils.R

@@ -23,14 +23,17 @@ feed_has_non_empty_table <- function(gtfs_obj, table_name) {
 
 #' Convert empty strings ("") to NA values in all gtfs tables
 #' 
+#' [read_gtfs()] converts all empty strings to `NA` values
+#' 
 #' @param gtfs_obj gtfs feed (tidygtfs object)
 #'  
 #' @return a gtfs_obj where all empty strings in tables have been replaced with NA
 #' @seealso [na_to_empty_strings()]
 #' @export
 empty_strings_to_na = function(gtfs_obj) {
   tbl_names = names(gtfs_obj)
-  tbl_names <- tbl_names[tbl_names != "."]
+  non_txt = names(gtfs_reference_filetype[gtfs_reference_filetype != "txt"])
+  tbl_names <- setdiff(tbl_names, c(".", non_txt))
   for(tbl in tbl_names) {
     if(inherits(gtfs_obj[[tbl]], "data.frame")) {
       gtfs_obj[[tbl]][gtfs_obj[[tbl]] == ""] <- NA
@@ -40,6 +43,8 @@ empty_strings_to_na = function(gtfs_obj) {
 }
 
 #' Convert NA values to empty strings ("")
+#' 
+#' [write_gtfs()] converts `NA` to empty strings
 #'  
 #' @param gtfs_obj gtfs feed (tidygtfs object)
 #' @return a gtfs_obj where all NA strings in tables have been replaced with ""

data/gtfs_duke.rda

@@ -93,25 +93,6 @@ test_that("filter_feed_by_date", {
   expect_s3_class(g2$stop_times, "tbl_df")
 })
 
-test_that("empty_strings_to_na", {
-  gpath = system.file("extdata", "sample-feed-fixed.zip", package = "tidytransit")
-  g1 = read_gtfs(gpath)
-  g_na = empty_strings_to_na(g1)
-  g2 = na_to_empty_strings(g_na)
-
-  for(tbl in names(g1)) {
-    expect_equal(g1[[tbl]], g2[[tbl]])
-  }
-  
-  tmppath = tempfile(fileext = ".zip")
-  write_gtfs(g_na, tmppath)
-  g3 = read_gtfs(tmppath)
-   
-  for(tbl in names(g1)) {
-    expect_equal(g1[[tbl]], g3[[tbl]])
-  } 
-})
-
 test_that("feed_contains, feed_has_non_empty_table", {
   g = gtfs_duke
   expect_false(feed_contains(g, "dates_services"))