## 12.2 Lab 11: Descriptive statistics

Generally, we can use datamanagement functions in the dplyr package to produce various descriptive statistics such as mean(), sd(), min(), max() across our dataset, across different groups. We’ll work with the tweets dataset again. You can find a description here.

library(DBI)
db <- dbConnect(RSQLite::SQLite(), "./www/tweets-sentiment-db.sqlite")
dbListTables(db)
## [1] "table_tweets"
dbListFields(db, "table_tweets")
## [1] "target" "ids"    "date"   "flag"   "user"   "text"
data <- dbGetQuery(db, "SELECT  target, ids, date, user, text
FROM table_tweets
ORDER BY RANDOM()
LIMIT 200000")
dbDisconnect(db)
nrow(data) # check how many rows there are
## [1] 200000

Get the number of tweets per user:

library(dplyr)
# Number per user
data %>% group_by(user) %>% dplyr::summarise(n.tweets = n()) %>% arrange(desc(n.tweets)) %>% slice(1:20)
user n.tweets
lost_dog 72
tweetpet 40
SongoftheOss 39
webwoke 38
VioletsCRUK 34
what_bugs_u 32
DarkPiano 31
nuttychris 31
Dogbook 30
SallytheShizzle 30
wowlew 30
enamoredsoul 29
keza34 29
Broooooke_ 28
twebbstack 27
Jayme1988 26
Karen230683 26
tsarnick 26

Aggregate the data by user and get the number of tweets as well as the average sentiment:

library(dplyr)
# Mean of target
# target: the polarity of the tweet (0 = negative, 2 = neutral, 4 = positive)
data %>% group_by(user) %>% dplyr::summarise(n.tweets = n(), mean.target = mean(target, na.rm=TRUE)) %>% arrange(desc(n.tweets)) %>% slice(1:20)
user n.tweets mean.target
lost_dog 72 0.0000000
tweetpet 40 0.0000000
SongoftheOss 39 2.0512821
webwoke 38 0.7368421
VioletsCRUK 34 3.1764706
what_bugs_u 32 4.0000000
DarkPiano 31 4.0000000
nuttychris 31 1.4193548
Dogbook 30 1.7333333
SallytheShizzle 30 1.3333333
wowlew 30 0.0000000
enamoredsoul 29 3.4482759
keza34 29 4.0000000
Broooooke_ 28 2.5714286
# Correlation between number of tweets and sentiment
cor(data.agg$n.tweets, data.agg$mean.target)
## [1] 0.03800391