
library(testthat)

source("user_code.R")
test_that("remove_outliers sets outliers in income to NA using IQR", {
    survey_test <- data.frame(income = c(1, 2, 3, 4, 1000, NA))
    out <- remove_outliers(survey_test$income)
    # 1000 should be NA, others not
    expect_true(is.na(out[5]), info = "Outlier value not set to NA as expected")
    expect_false(any(is.na(out[1:4])), info = "Non-outlier values incorrectly set to NA")
})

test_that("fix_gender standardizes gender values and sets unknowns to NA", {
    genders <- c("M", "female", "F", "MALE", "Woman", "man", "FEMALE", "m", "unknown", NA)
    fixed <- fix_gender(genders)
    expect_equal(fixed[1], "Male", info = "'M' not converted to 'Male'")
    expect_equal(fixed[2], "Female", info = "'female' not converted to 'Female'")
    expect_equal(fixed[5], "Female", info = "'Woman' not converted to 'Female'")
    # expect_equal(fixed[9], NA, info = "Unknown gender not set to NA")
    # expect_equal(fixed[10], NA, info = "NA input should remain NA")
})

test_that("ages less than 0 or greater than 120 are set to NA", {
    test_ages <- c(25, 130, -5, 50)
    df <- data.frame(age = test_ages)
    out <- df %>% mutate(age = ifelse(age < 0 | age > 120, NA, age))
    expect_true(is.na(out$age[2]), info = "Age > 120 not set to NA")
    expect_true(is.na(out$age[3]), info = "Negative age not set to NA")
    expect_equal(out$age[1], 25, info = "Valid age incorrectly modified")
})

test_that("rows with missing or invalid age or gender are removed", {
    survey <- data.frame(
      id = 1:4,
      age = c(25, NA, 30, 150),
      income = c(40000, 50000, 60000, 70000),
      gender = c("M", "female", NA, "unknown"),
      stringsAsFactors = FALSE
    )
    cleaned <- survey %>%
      mutate(age = ifelse(age < 0 | age > 120, NA, age)) %>%
      mutate(gender = fix_gender(gender)) %>%
      filter(!is.na(age) & !is.na(gender))
    expect_equal(nrow(cleaned), 1, info = "Rows with invalid/missing age or gender not removed correctly")
    expect_equal(cleaned$id, 1, info = "Incorrect row kept after filtering")
})

test_that("cleaned_survey contains only valid, cleaned rows for age, income, gender", {
    # Use the full pipeline and check the result
    survey <- data.frame(
      id = 1:10,
      age = c(25, 30, NA, 22, 150, 29, 27, 24, -5, 28),
      income = c(40000, 50000, 60000, NA, 70000, 80000, 90000, 1000000, 55000, 65000),
      gender = c("M", "female", "F", "MALE", "Woman", "man", "FEMALE", "m", "unknown", NA),
      stringsAsFactors = FALSE
    )
    cleaned <- survey %>%
      mutate(age = ifelse(age < 0 | age > 120, NA, age)) %>%
      mutate(income = remove_outliers(income)) %>%
      mutate(gender = fix_gender(gender)) %>%
      filter(!is.na(age) & !is.na(gender))
    # Only rows 1, 2, 4, 6, 7, 8, 10 are valid after cleaning (but 10 has NA gender)
    expect_true(all(cleaned$age > 0 & cleaned$age <= 120), info = "Invalid ages present after cleaning")
    expect_true(all(cleaned$gender %in% c("Male", "Female")), info = "Invalid genders present after cleaning")
    expect_false(any(is.na(cleaned$age)), info = "NA ages present after filtering")
    expect_false(any(is.na(cleaned$gender)), info = "NA genders present after filtering")
    # Check that income outlier (1000000) is NA
    expect_true(is.na(cleaned$income[cleaned$id == 8]), info = "Income outlier not set to NA")
})

test_main.R

Master the essential skills for ensuring data quality in R. Learn to detect and handle missing, duplicated, and inconsistent data, identify outliers, apply data validation rules, and build robust data cleaning pipelines. This course combines engaging theory with hands-on challenges to prepare you for real-world data cleaning tasks.

Explore the foundations of data cleaning and quality assurance in R. Each chapter introduces a core concept, followed by a hands-on challenge to reinforce your learning.

Challenge: Build a Cleaning Pipeline for Survey Data

Ratkaisu

Challenge: Build a Cleaning Pipeline for Survey Data

Ratkaisu