5 Packages & data downloads

5.1 Install & import DWQ’s R packages

devtools::install_github('utah-dwq/wqTools')
devtools::install_github('utah-dwq/irTools')

library(wqTools)
library(irTools)

5.2 Download and import data

5.2.1 Data download

downloadWQP(outfile_path='01-raw-data',start_date='10/01/2008', end_date='09/30/2014', zip=TRUE, unzip=TRUE, retrieve=c("narrowresult", "activity", "detquantlim"))
downloadWQP(outfile_path='01-raw-data', zip=FALSE, retrieve="sites")

Note– Having an issue downloading sites for POR date range (Unclear why exactly. Seems to be associated applying date query parameters to older sites). Downloading all sites separately, then subsetting to just those included in narrowresult.

sites=read.csv(file='01-raw-data/sites-2019-04-04.csv')
nr=read.csv(file='01-raw-data/narrowresult-2019-04-04.csv')
sites=sites[sites$MonitoringLocationIdentifier %in% nr$MonitoringLocationIdentifier,]
write.csv(file='01-raw-data/sites-2019-04-04.csv', sites, row.names=F)
rm(sites)

5.2.2 Data imports

Read raw data into R, remove duplicates and check for orphans

irdata <- readWQPFiles(file_select=FALSE,
            narrowresult_file = "01-raw-data\\narrowresult-2019-04-04.csv",
            sites_file = "01-raw-data\\sites-2019-04-04.csv",
            activity_file = "01-raw-data\\activity-2019-04-04.csv",
            detquantlim_file = "01-raw-data\\detquantlim-2019-04-04.csv",
            orph_check = TRUE)

## [1] "------------READING IN FILES--------------"
## [1] "----REMOVING EXACT DUPLICATES-----"
## [1] "-----PERFORMING ORPHAN RECORD CHECKS------"
## [1] "3 orphan records detected in sites file with no match to narrowresult."
## [1] "narrowresult_sites_orphans object created containing orphan records."
## [1] "Date forms between narrowresult and activity often cause erroneous orphans. Check date forms below. If date forms do not match, prior conversion using as.Date() is needed."
## [1] "narrowresult file:"
## [1] 2010-07-13 2010-07-13 2010-07-13 2010-07-13 2010-07-13 2010-07-13
## 1775 Levels: 2008-10-01 2008-10-02 2008-10-03 2008-10-04 ... 2014-09-30
## [1] "activity file:"
## [1] 2010-07-13 2010-07-13 2010-07-13 2010-07-13 2010-07-13 2010-07-13
## 1777 Levels: 2008-10-01 2008-10-02 2008-10-03 2008-10-04 ... 2014-09-30
## [1] "98 orphan records detected in activity file with no match to narrowresult."
## [1] "narrowresult_activity_orphans object created containing orphan records."
## [1] "NOTE: narrowresult will likely have many orphan records not represented in detquantlim. This occurs for a few reasons: (1) labs sometimes do not report detection quantitation limits, and (2) field measurements often do not report detection quantitation limits."
## [1] "684950 orphan records detected in narrowresult file with no match to detquantlim."
## [1] "narrowresult_detquantlim_orphans object created containing orphan records."
## [1] "Need to figure out non-numeric data in numeric columns conundrum."

## Warning in wqTools::facToNum(wqpdat$merged_results$ResultMeasureValue): NAs
## introduced by coercion

## [1] "----FILES SUCCESSFULLY ADDED TO R OBJECT LIST----"

objects(irdata)

## [1] "detquantlim"                      "merged_results"                  
## [3] "narrowresult_activity_orphans"    "narrowresult_detquantlim_orphans"
## [5] "narrowresult_site_orphans"        "sites"