Verasight | Checking for potential pop-up blocker issue

Data

  • Filtered out suspects (=potential bots)
  • Filtered in progress 100% and 14% cases (Total N=71)

    • 14% (N=34)

    • 100% (N=37)

library(readr)
library(tidyverse)
library(stringr)

# Load data
dta <- read_csv("raw-responses_2024-01-17.csv")
Suspects_Sheet1 <- read_csv("Suspects - Sheet1.csv")

names(Suspects_Sheet1) = c("userid", "screename", "vsid", "notes")

Suspects_Sheet1 |>
  mutate(flag = "suspects") -> Suspects_Sheet1

dta |>
  left_join(Suspects_Sheet1[,c(2,5)], by="screename") |>
  filter(is.na(flag)) -> nosus

nosus |>
  filter(progress == 100 | progress == 14) |>
  filter(!(progress == 100 & is.na(screename))) |>
  select(c(progress, metainfo_browser:metainfo_resolution, userid, screename, vsid, feedback)) -> dta2

Result

It seems that using phone is related with drop out on the authorization page.

  • 14% group: Safari iPhone ↑

  • 100% group: Chrome + Windows ↑

(ratio indicates % within each group)

dta2 |>
  mutate(
    OS = ifelse(str_detect(dta2$metainfo_operating_system, "Android"), "Android", metainfo_operating_system),
    Browser = metainfo_browser
  ) -> dta2

dta2 |>
  group_by(progress) |>
  count(Browser) |>
  mutate(ratio = round(ifelse(progress==14, n/34, n/37), 2)) 
## # A tibble: 8 × 4
## # Groups:   progress [2]
##   progress Browser           n ratio
##      <dbl> <chr>         <int> <dbl>
## 1       14 Chrome           19  0.56
## 2       14 Safari            1  0.03
## 3       14 Safari iPhone    14  0.41
## 4      100 Chrome           23  0.62
## 5      100 Edge              2  0.05
## 6      100 Firefox           1  0.03
## 7      100 Safari            3  0.08
## 8      100 Safari iPhone     8  0.22
dta2 |>
  group_by(progress, Browser) |>
  count(OS) |>
  mutate(ratio = round(ifelse(progress==14, n/34, n/37), 2))
## # A tibble: 12 × 5
## # Groups:   progress, Browser [8]
##    progress Browser       OS                  n ratio
##       <dbl> <chr>         <chr>           <int> <dbl>
##  1       14 Chrome        Android            16  0.47
##  2       14 Chrome        Macintosh           1  0.03
##  3       14 Chrome        Windows NT 10.0     1  0.03
##  4       14 Chrome        q2q                 1  0.03
##  5       14 Safari        Macintosh           1  0.03
##  6       14 Safari iPhone iPhone             14  0.41
##  7      100 Chrome        Android            13  0.35
##  8      100 Chrome        Windows NT 10.0    10  0.27
##  9      100 Edge          Windows NT 10.0     2  0.05
## 10      100 Firefox       Android             1  0.03
## 11      100 Safari        Macintosh           3  0.08
## 12      100 Safari iPhone iPhone              8  0.22
library(ggplot2)
library(ggthemes)
library(scales)
library(grid)
library(gridExtra)

blank_theme <- theme_classic()+
  theme(
  axis.title.x = element_blank(),
  axis.title.y = element_blank(),
  panel.border = element_blank(),
  axis.text.x = element_blank(),
  panel.grid=element_blank(),
  axis.ticks = element_blank(),
  plot.title=element_text(size=12, face="bold")
  )

dta2 |>
  group_by(progress) |>
  filter(progress==14) |>
  ungroup() |>
  select(Browser) |>
  count(Browser) |>
  ggplot(aes(x = "", y = n, fill = Browser)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  geom_text(aes(label = percent(n/100)),
            size=4,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 14%") -> p1

dta2 |>
  group_by(progress) |>
  filter(progress==100) |>
  ungroup() |>
  select(Browser) |>
  count(Browser) |>
  ggplot(aes(x = "", y = n, fill = Browser)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  theme(axis.text.x=element_blank())+
  geom_text(aes(label = percent(n/100)),
            size=4,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 100%") -> p2

dta2 |>
  group_by(progress) |>
  filter(progress==14) |>
  ungroup() |>
  select(OS) |>
  count(OS) |>
  ggplot(aes(x = "", y = n, fill = OS)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  geom_text(aes(label = percent(n/100)),
            size=3,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 14%") -> p3

dta2 |>
  group_by(progress) |>
  filter(progress==100) |>
  ungroup() |>
  select(OS) |>
  count(OS) |>
  ggplot(aes(x = "", y = n, fill = OS)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  theme(axis.text.x=element_blank())+
  geom_text(aes(label = percent(n/100)),
            size=3,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 100%") -> p4

grid.arrange(p1, p2, ncol=2)

grid.arrange(p3, p4, ncol=2)

What if we compare only those who used their phones to take the survey? Are there noticeable differences between 100% vs. 14% groups?

dta2 |>
  group_by(progress) |>
  mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
                         "Android Chrome", NA)) |>
  select(progress, groups, Browser) |>
  filter(groups == "Android Chrome" |
           Browser == "Safari iPhone") |>
  mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
  select(!Browser) |>
  group_by(progress, groups) |>
  count() |>
  mutate(ratio = round(ifelse(progress==14, n/30, n/21), 2)) 
## # A tibble: 4 × 4
## # Groups:   progress, groups [4]
##   progress groups             n ratio
##      <dbl> <chr>          <int> <dbl>
## 1       14 Android Chrome    16  0.53
## 2       14 Safari iPhone     14  0.47
## 3      100 Android Chrome    13  0.62
## 4      100 Safari iPhone      8  0.38
dta2 |>
  group_by(progress) |>
  mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
                         "Android Chrome", NA)) |>
  select(progress, groups, Browser) |>
  filter(groups == "Android Chrome" |
           Browser == "Safari iPhone") |>
  mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
  select(!Browser) |>
  filter(progress==14) |>
  count(groups) |>
  ggplot(aes(x = "", y = n, fill = groups)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  geom_text(aes(label = percent(n/100)),
            size=4,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 14%") -> pn1


dta2 |>
  group_by(progress) |>
  mutate(groups = ifelse(Browser=="Chrome" & OS=="Android",
                         "Android Chrome", NA)) |>
  select(progress, groups, Browser) |>
  filter(groups == "Android Chrome" |
           Browser == "Safari iPhone") |>
  mutate(groups = ifelse(is.na(groups), "Safari iPhone", groups)) |>
  select(!Browser) |>
  filter(progress==100) |>
  count(groups) |>
  ggplot(aes(x = "", y = n, fill = groups)) +
  geom_bar(width = 1, stat = "identity") +
  coord_polar("y", start = 0) +
  blank_theme +
  scale_fill_tableau() +
  geom_text(aes(label = percent(n/100)),
            size=4,
            position=position_stack(vjust = 0.5)) +
  ggtitle("Progress == 100%") -> pn2

grid.arrange(pn1, pn2, ncol=2)

Conclusion

Although we need to wait for Verasight data on what those respondents say about the reasons they left at the authorization page, I could infer the following from this analysis:

  1. Using phones might be the main factor.
    • Among 14% group (N=34), 30 people used phone (88.2%)

    • Among 100% group (N=37), 21 people used phone (56.8%)

    • However, we cannot conclude that pop-up blocker is the issue.

    • It might be that on the web browsers in desktop/laptops, people are already logged in to Twitter, which makes the authorization step much easier.

  2. Among those who used phones, iPhone seems to be trickier.
    • Although the absolute number is lower compared to Android users, iPhone users who took their surveys on iPhone Safari tended to drop out more on the authorization page. (14 out of 22 people)

    • In the feedback (open-ended question), one participant in the 100% group mentioned that “links acted a little funky on iPhone”. (See below)

    • We need to check how pop-up blockers and other features work differently between iPhone Safari and Android Chrome.

library(DT)
feedback = as.data.frame(dta2$feedback) |> filter(!is.na(`dta2$feedback`))
datatable(feedback)

To take some interesting ones and categorize:

# Potential error
[9] "The slider on familiarity with the last few sources wouldn’t let me rate no option"
[116] "Awesome survey, links acted a little funky on IPhone."

# X
[17] "Tiktok is preferred world news outlet.X NEEDS TO ANTE UP,SO TO SPEAK..."

# Intent of survey
[18] "Interesting survey.  I'm trying to understand the nature of this and what its trying to accomplish."
[29] "Survey was comfortable. I feel it is an attempt to sway election outcome."