4. Scholar

1 getting started

#start with clean workspace 
rm(list=ls())
getwd()

2 packages

library(data.table) 
library(tidyverse) 
require(xml2)
require(rvest)
require(devtools)
require(scholar)
require(stringi)

3 Scholar

“https://scholar.google.nl/scholar/citations?view_op=search_authors&mauthors=jochem+tolsma&hl=en&oi=ao”

https://scholar.google.nl/citations?view_op=search_authors&mauthors=lonneke+van den berg&hl=en&oi=ao

get_scholar_resp(“https://scholar.google.nl/citations?view_op=search_authors&mauthors=lonneke+van den berg&hl=en&oi=ao”)

get_scholar_resp(“https://scholar.google.nl/citations?view_op=search_authors&mauthors=lonneke+van+den+berg&hl=en&oi=ao”)

3.1 fixing a bug in the get_scholar_id function.

get_scholar_id_fix <- function (last_name = "", first_name = "", affiliation = NA)
{
  if (!any(nzchar(c(first_name, last_name))))
    stop("At least one of first and last name must be specified!")
  site <- getOption("scholar_site")
  url <- paste0(site, "/citations?view_op=search_authors&mauthors=",
                first_name, "+", last_name, "&hl=en&oi=ao")
  page <- get_scholar_resp(url)
  if (is.null(page))
    return(NA)
  aa <- httr::content(page, as = "text")
  # added by Bas Hofstra: bugfix for IDs that have a dash ("-")
  ids <- substring(aa, regexpr(";user=", aa))
  ids <- substr(ids, 1, 19) # error prone, but unsure how to solve otherwise
  # if (nchar(stringr::str_extract_all(string = aa, pattern = ";user=[[:alnum:]]+[[:punct:]]")[[1]][1]) < 18) {
  #   ids <- stringr::str_extract_all(string = aa, pattern = ";user=[[:alnum:]]+[[:punct:]]+[[:alnum:]]+[[:punct:]]")
  # } else {
  #   ids <- stringr::str_extract_all(string = aa, pattern = ";user=[[:alnum:]]+[[:punct:]]")
  # }
  if (length(unlist(ids)) == 0) {
    message("No Scholar ID found.")
    return(NA)
  }
  ids <- ids %>% unlist %>% gsub(";user=|[[:punct:]]$", "",
                                 .) %>% unique
  if (length(ids) > 1) {
    profiles <- lapply(ids, scholar::get_profile)
    if (is.na(affiliation)) {
      x_profile <- profiles[[1]]
      warning("Selecting first out of ", length(profiles),
              " candidate matches.")
    }
    else {
      which_profile <- sapply(profiles, function(x) {
        stringr::str_count(string = x$affiliation, pattern = stringr::coll(affiliation,
                                                                           ignore_case = TRUE))
      })
      if (all(which_profile == 0)) {
        warning("No researcher found at the indicated affiliation.")
        return(NA)
      }
      else {
        x_profile <- profiles[[which(which_profile !=
                                       0)]]
      }
    }
  }
  else {
    x_profile <- scholar::get_profile(id = ids)
  }
  return(x_profile$id)
}

3.2 scholars id.

Don’t forget to check manually if everything is okay.

We make an important decisson here. we remove staff members without scholar ids. perhaps a bit strange for our RSiena analysis later. I mean, we can include isolates

YOU MAY NEED TO ADD AN TRYCATCH TO THIS LOOP AS WELL TO AVOID TIME OUT ERRORS, SEE THE NEXT LOOP BELOW

load("names_df_v20221006.RData") 
cs_df <- names_df
cs_df$gs_id <- "" #don't make it empty everytime you run scrape! 
cs_df$affiliation[cs_df$affiliation=="RU"] <- "Radboud University"
cs_df$affiliation[cs_df$affiliation=="UU"] <- "Utrecht University"
cs_df$last_name3 <- str_replace_all(cs_df$last_name, " ", "+")  

time <- .1 # I placed the waiting time outside the loop
i <- 1 # Our loop iterator is now a variable. This means I can change it within a while loop. Using a for loop you cant change your iterator in the loop itself.

while (i <= nrow(cs_df)) {
  print(i)
  Sys.sleep(time)
  if (cs_df[i,c("gs_id")]!="") {
    i <- i + 1
    next
  }
  tryCatch({
     if (cs_df[i,c("gs_id")]=="") {
       cs_df[i,c("gs_id")] <- get_scholar_id_fix(last_name = cs_df[i, c("last_name3")], first_name = cs_df[i, c("first_name")],affiliation = cs_df[i,c("affiliation")])
      i <- i + 1
      time <- 1 #reset time if succesfull 
     }
    }, 
    warning = function(w) {
        cat("WARNING:", conditionMessage(w), "\n") #WARNING message
        cs_df[i,c("gs_id")] <<- ""
        i <<- i + 1
        time <<- .1 #reset time if succesfull 
        }, 
    error=function(e){
      err <- conditionMessage(e)
      if (stringr::str_detect(err, "Google is rate limiting you")) {
        time <<- min(time + 100, 3600*2)
        cat("Error:", conditionMessage(e), "\n") #ERROR message
        cat("sleep time:", time,  "\n")
        cat("ik zit in loop", i)
      } else {
        cat("ERROR :", conditionMessage(e), "\n") # continue on error, but print the error
        i <<- i + 1
        time <<- .1 #reset time if succesfull 
        }
      })
}

let us repeat without affiliation.

scholartemp <- cs_df
save(scholartemp, file="./data/scholartemp_v20221006.RData")
rm(scholartemp)
rm(names_df)

4 publications and profiles

cs_df <- cs_df[!cs_df$gs_id == "", ] 
cs_df <- cs_df[!is.na(cs_df$gs_id), ] 

soc_list_profiles <- list()  # first we create an empty list that we then fill up with the for loop
soc_list_publications <- list()

time <- .1 # I placed the waiting time outside the loop
i <- 1 # Our loop iterator is now a variable. This means I can change it within a while loop. Using a for loop you cant change your iterator in the loop itself.

while (i <= nrow(cs_df)) {
    print(i)
    Sys.sleep(time)

    tryCatch({
    soc_list_profiles[[i]] <- get_profile(cs_df[i, c("gs_id")]) 
    soc_list_publications[[i]] <- get_publications(cs_df[i, c("gs_id")])
    soc_list_publications[[i]][, c("gs_id")] <- cs_df[i, c("gs_id")]  
    i <- i + 1 
    time <- .1
    },
    
      warning = function(w) {
        cat("WARNING:", conditionMessage(w), "\n") 
        i <<- i + 1
        }, 
    
      error =function(e) {
        time <<- min(time + 100, 3600*2)
        cat("Error:", conditionMessage(e), "\n") 
        cat("sleep time:", time,  "\n")
        cat("ik zit in loop", i)
      })
}

cs_df_publications <- names_df_publications <- bind_rows(soc_list_publications)
cs_list_profiles <- names_list_profiles <- soc_list_profiles

save(names_df_publications, file="./data/names_df_publications_v20221006.RData")
save(names_list_profiles, file= "./data/names_list_profiles_v20221006.RData")

4.1 put the info of the profiles in our data set of staff members cs_df

cs_profiles_df <- list()


for (i in 1:length(cs_list_profiles)) {
    # soc_profiles_df[[i]] <- data.frame(t(unlist(soc_list_profiles[[i]][1:8]))) #some annyoing
    # data handling
  if (!is.null(cs_list_profiles[[i]])) {
    cs_profiles_df[[i]] <- unlist(cs_list_profiles[[i]][1:8])
    cs_profiles_df[[i]] <- data.frame(cs_profiles_df[[i]])
    cs_profiles_df[[i]] <- t(cs_profiles_df[[i]])
    row.names(cs_profiles_df[[i]]) <- NULL
    cs_profiles_df[[i]] <- data.frame(cs_profiles_df[[i]])
  }
}

let op we hebben dus duplicates. mensen staan zowel bij computer science als bij data science.

cs_profiles_df2 <- bind_rows(cs_profiles_df)

names(cs_profiles_df2)[1] <- "gs_id"
#cs_df <- names_df_copy[!names_df_copy$gs_id=="",]
length(cs_df$id)
duplicated(cs_df$gs_id)
cs_df[order(cs_df$gs_id),]
cs_df2 <- data.frame(cbind(cs_df, cs_profiles_df2) )
names_df <- cs_df2

save(names_df, file="./data/names_df2_v20220106.RData")

5 citation history

cs_df <- cs_df2

# get citation history of a scholar
cs_staff_cit <- list()

time <- .1 # I placed the waiting time outside the loop
i <- 1 # Our loop iterator is now a variable. This means I can change it within a while loop. Using a for loop you cant change your iterator in the loop itself.

while (i <= nrow(cs_df)) {
    print(i)
    Sys.sleep(time)

    tryCatch({
      cs_staff_cit[[i]] <- get_citation_history(cs_df[i, c("gs_id")])
        if (nrow(cs_staff_cit[[i]]) > 0) {
          cs_staff_cit[[i]][, c("gs_id")] <- cs_df[i, c("gs_id")]  # again attach the gs_id as third column
        }
    i <- i + 1
    time <- .1
    },
      warning = function(w) {
        cat("WARNING:", conditionMessage(w), "\n") #WARNING message
        i <<- i + 1}, #BUT WE DO WANT TO CONTINUE. NOTE THE DOUBLE << THIS IS BECAUSE I WANT TO CHANGE A VARIABLE WHICH EXISTS OUTSIDE THE WARNING FUNCTION
      error =function(e) {
        time <<- min(time + 10, 3600*2)
        cat("Error:", conditionMessage(e), "\n") #ERROR message
        cat("sleep time:", time,  "\n")
        cat("ik zit in loop", i)
        #AFTER THE NEW SLEEP TIME, WE TRY AGAIN, WE THEREFORE DO NOT UPDATE i. ideally you also want to have some break option. And maybe you also want to save your data when you hit a time out error. 
      })

    
}
names_staff_cit <- bind_rows(cs_staff_cit)

save(names_staff_cit, file="./data/names_staff_cit_v20221006.RData")