Verasight | Checking for potential pop-up blocker issue
Do Won Kim
2024-01-20
Data
- Filtered out suspects (=potential bots)
Filtered in progress 100% and 14% cases (Total N=71)
14% (N=34)
100% (N=37)
library(readr)
library(tidyverse)
library(stringr)
# Load data
dta <- read_csv("raw-responses_2024-01-17.csv")
Suspects_Sheet1 <- read_csv("Suspects - Sheet1.csv")
names(Suspects_Sheet1) = c("userid", "screename", "vsid", "notes")
Suspects_Sheet1 |>
mutate(flag = "suspects") -> Suspects_Sheet1
dta |>
left_join(Suspects_Sheet1[,c(2,5)], by="screename") |>
filter(is.na(flag)) -> nosus
nosus |>
filter(progress == 100 | progress == 14) |>
filter(!(progress == 100 & is.na(screename))) |>
select(c(progress, metainfo_browser:metainfo_resolution, userid, screename, vsid, feedback)) -> dta2Result
It seems that using phone is related with drop out on the authorization page.
14% group: Safari iPhone ↑
100% group: Chrome + Windows ↑
(ratio indicates % within each group)
dta2 |>
mutate(
OS = ifelse(str_detect(dta2$metainfo_operating_system, "Android"), "Android", metainfo_operating_system),
Browser = metainfo_browser
) -> dta2
dta2 |>
group_by(progress) |>
count(Browser) |>
mutate(ratio = round(ifelse(progress==14, n/34, n/37), 2)) ## # A tibble: 8 × 4
## # Groups: progress [2]
## progress Browser n ratio
## <dbl> <chr> <int> <dbl>
## 1 14 Chrome 19 0.56
## 2 14 Safari 1 0.03
## 3 14 Safari iPhone 14 0.41
## 4 100 Chrome 23 0.62
## 5 100 Edge 2 0.05
## 6 100 Firefox 1 0.03
## 7 100 Safari 3 0.08
## 8 100 Safari iPhone 8 0.22dta2 |>
group_by(progress, Browser) |>
count(OS) |>
mutate(ratio = round(ifelse(progress==14, n/34, n/37), 2))## # A tibble: 12 × 5
## # Groups: progress, Browser [8]
## progress Browser OS n ratio
## <dbl> <chr> <chr> <int> <dbl>
## 1 14 Chrome Android 16 0.47
## 2 14 Chrome Macintosh 1 0.03
## 3 14 Chrome Windows NT 10.0 1 0.03
## 4 14 Chrome q2q 1 0.03
## 5 14 Safari Macintosh 1 0.03
## 6 14 Safari iPhone iPhone 14 0.41
## 7 100 Chrome Android 13 0.35
## 8 100 Chrome Windows NT 10.0 10 0.27
## 9 100 Edge Windows NT 10.0 2 0.05
## 10 100 Firefox Android 1 0.03
## 11 100 Safari Macintosh 3 0.08
## 12 100 Safari iPhone iPhone 8 0.22library(ggplot2)
library(ggthemes)
library(scales)
library(grid)
library(gridExtra)
blank_theme <- theme_classic()+
theme(
axis.title.x = element_blank(),
axis.title.y = element_blank(),
panel.border = element_blank(),
axis.text.x = element_blank(),
panel.grid=element_blank(),
axis.ticks = element_blank(),
plot.title=element_text(size=12, face="bold")
)
dta2 |>
group_by(progress) |>
filter(progress==14) |>
ungroup() |>
select(Browser) |>
count(Browser) |>
ggplot(aes(x = "", y = n, fill = Browser)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
geom_text(aes(label = percent(n/100)),
size=4,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 14%") -> p1
dta2 |>
group_by(progress) |>
filter(progress==100) |>
ungroup() |>
select(Browser) |>
count(Browser) |>
ggplot(aes(x = "", y = n, fill = Browser)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
theme(axis.text.x=element_blank())+
geom_text(aes(label = percent(n/100)),
size=4,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 100%") -> p2
dta2 |>
group_by(progress) |>
filter(progress==14) |>
ungroup() |>
select(OS) |>
count(OS) |>
ggplot(aes(x = "", y = n, fill = OS)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
geom_text(aes(label = percent(n/100)),
size=3,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 14%") -> p3
dta2 |>
group_by(progress) |>
filter(progress==100) |>
ungroup() |>
select(OS) |>
count(OS) |>
ggplot(aes(x = "", y = n, fill = OS)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
theme(axis.text.x=element_blank())+
geom_text(aes(label = percent(n/100)),
size=3,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 100%") -> p4
grid.arrange(p1, p2, ncol=2)
grid.arrange(p3, p4, ncol=2)
What if we compare only those who used their phones to take the survey? Are there noticeable differences between 100% vs. 14% groups?
dta2 |>
group_by(progress) |>
mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
"Android Chrome", NA)) |>
select(progress, groups, Browser) |>
filter(groups == "Android Chrome" |
Browser == "Safari iPhone") |>
mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
select(!Browser) |>
group_by(progress, groups) |>
count() |>
mutate(ratio = round(ifelse(progress==14, n/30, n/21), 2)) ## # A tibble: 4 × 4
## # Groups: progress, groups [4]
## progress groups n ratio
## <dbl> <chr> <int> <dbl>
## 1 14 Android Chrome 16 0.53
## 2 14 Safari iPhone 14 0.47
## 3 100 Android Chrome 13 0.62
## 4 100 Safari iPhone 8 0.38dta2 |>
group_by(progress) |>
mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
"Android Chrome", NA)) |>
select(progress, groups, Browser) |>
filter(groups == "Android Chrome" |
Browser == "Safari iPhone") |>
mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
select(!Browser) |>
filter(progress==14) |>
count(groups) |>
ggplot(aes(x = "", y = n, fill = groups)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
geom_text(aes(label = percent(n/100)),
size=4,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 14%") -> pn1
dta2 |>
group_by(progress) |>
mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
"Android Chrome", NA)) |>
select(progress, groups, Browser) |>
filter(groups == "Android Chrome" |
Browser == "Safari iPhone") |>
mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
select(!Browser) |>
filter(progress==100) |>
count(groups) |>
ggplot(aes(x = "", y = n, fill = groups)) +
geom_bar(width = 1, stat = "identity") +
coord_polar("y", start = 0) +
blank_theme +
scale_fill_tableau() +
geom_text(aes(label = percent(n/100)),
size=4,
position=position_stack(vjust = 0.5)) +
ggtitle("Progress == 100%") -> pn2
grid.arrange(pn1, pn2, ncol=2)
Conclusion
Although we need to wait for Verasight data on what those respondents say about the reasons they left at the authorization page, I could infer the following from this analysis:
- Using phones might be the main factor.
Among 14% group (N=34), 30 people used phone (88.2%)
Among 100% group (N=37), 21 people used phone (56.8%)
However, we cannot conclude that pop-up blocker is the issue.
It might be that on the web browsers in desktop/laptops, people are already logged in to Twitter, which makes the authorization step much easier.
- Among those who used phones, iPhone seems to be trickier.
Although the absolute number is lower compared to Android users, iPhone users who took their surveys on iPhone Safari tended to drop out more on the authorization page. (14 out of 22 people)
In the feedback (open-ended question), one participant in the 100% group mentioned that “links acted a little funky on iPhone”. (See below)
We need to check how pop-up blockers and other features work differently between iPhone Safari and Android Chrome.
library(DT)
feedback = as.data.frame(dta2$feedback) |> filter(!is.na(`dta2$feedback`))
datatable(feedback)To take some interesting ones and categorize:
# Potential error
[9] "The slider on familiarity with the last few sources wouldn’t let me rate no option"
[116] "Awesome survey, links acted a little funky on IPhone."
# X
[17] "Tiktok is preferred world news outlet.X NEEDS TO ANTE UP,SO TO SPEAK..."
# Intent of survey
[18] "Interesting survey. I'm trying to understand the nature of this and what its trying to accomplish."
[29] "Survey was comfortable. I feel it is an attempt to sway election outcome."