3. Origin

1 getting started

#start with clean workspace 
rm(list=ls())
load("./data/names_df_v20221005.RData")

2 packages

library(data.table) 
library(tidyverse) 
require(xml2)
require(rvest)
require(devtools)
require(scholar)
require(stringi)

andere strategie. alle nobiliary particles er af. dan zonder deze door databank. dan indien geen info, via particles NL-identiteit (of spaans of Duits of arabisch)

lastname_df <- names_df
lastname_df$lastname2 <- as.character(str_split(lastname_df$lastname, pattern=",", n = 2, simplify = TRUE)[,1])
lastname_df$np <- as.character(str_split(lastname_df$lastname, pattern=",", n = 2, simplify = TRUE)[,2])

# creating URLs: origin
flaname <- function(x){
    paste(toupper(substring(x, 1, 1)),
          tolower(substring(x, 2, nchar(x))),
          sep = "")
}
lastname_df$lastname2 <- flaname(lastname_df$lastname2)

lastname_df$name_origin <- paste0("https://www.cbgfamilienamen.nl/nfb/detail_naam.php?gba_naam=",
                                lastname_df$lastname2,
                                "&nfd_naam=",
                                lastname_df$lastname2,
                                "&info=analyse+en+verklaring&operator=eq&taal=")

https://www.cbgfamilienamen.nl/nfb/detail_naam.php?gba_naam=tolsma&gba_naam=Tolsma&nfd_naam=&info=analyse+en+verklaring&operator=eq&taal=

#lastname_df$name_origin[14] https://www.cbgfamilienamen.nl/nfb/detail_naam.php?gba_naam= Tolsma &nfd_naam= Tolsma &info=analyse+en+verklaring&operator=eq&taal=

#https://www.cbgfamilienamen.nl/nfb/detail_naam.php?gba_lcnaam=kraaykamp&gba_naam=Kraaykamp&nfd_naam=Kraaijkamp%20(y)&operator=eq&taal=

hier slaan we alles op

name_originl <- list()
table_originl <- list()
time <- 0.1

3 crucical scrape loop

for (i in 1:nrow(lastname_df)) {
  print(i)
  Sys.sleep(time)
  tryCatch({ 
    name_originl[[i]]  <- read_html(lastname_df[i, c("name_origin")])
    table_originl[[i]] <- name_originl[[i]] %>% html_table()
  }, 
    warning = function(w) {
        cat("WARNING:", conditionMessage(w), "\n") #WARNING message
    },
    error=function(e){
      err <- conditionMessage(e)
      cat("Error:", conditionMessage(e), "\n") #ERROR message
    }  
  )
}

en vanaf hier is het eigenlijk alleen maar opschonen.

origin_txt <- list()
for (i in 1:length(name_originl)) {
    origin_txt [[i]] <- name_originl[[i]] %>% html_text() %>% as.character()
}

# Get out the relevant origin information from the xml lists
origin_ln <- list()

for (i in 1:length(name_originl)) {
  origin_ln[[i]] <- name_originl[[i]] %>% html_nodes("div") %>% rvest::html_text()
  origin_ln[[i]] <- origin_ln[[i]][[3]]
}

# Remove mess
for (i in 1:length(origin_ln)) {
  origin_ln[[i]] <- gsub("\\t", " ", origin_ln[[i]])
  origin_ln[[i]] <- gsub("\\n", " ", origin_ln[[i]])
}

# Flatten nested structure of the origin information
#origin_ln <- rbind(flatten(origin_ln))

# Detaching the names and origin info for easier data handling
origin <- unlist(origin_ln)



origin <- str_extract_all(origin, "varianten(.*?)©")

# Origin information is usually mentioned after "verklaring" or "kenmerken"
origin <- str_remove_all(origin, "varianten")
origin <- str_remove_all(origin, "CBG Bronnen")
origin <- str_remove_all(origin, "catalogus")
origin <- str_remove_all(origin, "©")


verklaring <- str_remove_all(origin, "kenmerken:(.*?)$")
kenmerken <- str_extract_all(origin, "kenmerken:(.*?)$")
kenmerken <- str_remove_all(kenmerken, "specifieke componenten:(.*?)$")
sc <- str_extract_all(origin, "specifieke componenten:(.*?)$") # Not directly relevant to us, but does mean that the name has a webpage


# Make into a neat dataframe with the names attached
verklaring <- trimws(verklaring, which = "both")
kenmerken <- trimws(kenmerken, which = "both")
sc <- trimws(sc, which = "both")
vk <- data.frame(lastname_df$id,lastname_df$lastname, verklaring, kenmerken, sc)

4 Separating names with Dutch & unknown origin

Next, we identify those names for which no additional information was found. This is important to distinguish Dutch names from names with unknown origins.

• Dutch names: no label indicating that the name is Dutch, but some other information available on name origin
• Unknown names: web page cannot be found, so origin information is empty.

# Identify last names that could not be found
vk <- vk %>%
  mutate(verklaring = ifelse(verklaring=="", 0, verklaring), 
         kenmerken = ifelse(kenmerken=="character(0)", 0, kenmerken),
         sc = ifelse(sc=="character(0)", 0, sc),
         no_info = nchar(verklaring) + nchar(kenmerken) + nchar(sc))

vk <- vk %>%
  mutate(no_info = ifelse(no_info==3, 1, 0), 
         verklaring = ifelse(verklaring==0, NA, verklaring),
         kenmerken = ifelse(kenmerken==0, NA, kenmerken))
# If there is no text in verklaring or kenmerken, the name could not be found in the databases. 

5 Extracting specific origin information

There are three main ways to get information about the origin of last names:

1. Under “kenmerken”, last names are assigned clickable tags. These tags include unspecified foreign name tags (“andere taal”), as well as specific foreign origins of the name (“Franse naam”, “Indische naam”). -> origin1 + origin4

2. Several names have more extensively written out stories behind the name, under “verklaring”. A number of names contain detailed (either country-level or regional) origins, usually in the form of “De naam [xyz] is afkomstig uit [country]”. -> origin2

3. Some names have origin information under “verklaring” in the form of the linguistic origins of the name. This can be country specific (e.g. Chinese name), but it can also apply to multiple countries when the language is spoken in more than 1 countries (e.g. Spanish name). -> origin3

In het script hieronder gaat ergens iets fout. Daarom heb ik etniciteit uiteindelijk met de hand uitgerekend.

# Step 1: extracting origin tags from kenmerken
vk <- vk %>%
  mutate(origin1 = str_extract(kenmerken, "[:upper:]([:lower:]{2,}) naam"))

# Note: sometimes multiple origins are mentioned. Currently, I only extract the first one. Otherwise, we should use str_extract_all. 



# Step 2: extracting origin info from verklaring 
vk <- vk %>%
  mutate(origin2 = ifelse(as.numeric(str_detect(verklaring, "afkomstig uit")) == 1, 
         str_remove(verklaring, ".*afkomstig uit"), NA))


# Step 3: extracting additional origin info from verklaring
vk <- vk %>%
  mutate(origin3 = str_extract(verklaring, "[:upper:]([:lower:]{2,}) (achter)?(familie)?(beroeps)?naam"))



# Finally, we clean up the origin information extracted above

# Origin1: already neat
vk$origin1 <- str_remove(vk$origin1, "Joodse naam") # can be Dutch & non-Dutch

# Origin2: messy
vk$origin2 <- str_remove(vk$origin2, "\\..*") # remove extra info in the following sentence 
vk$origin2 <- str_remove(vk$origin2, "\\;.*") # remove extra info in the following sentence 
vk$origin2 <- str_remove(vk$origin2, "\\(.*") # remove extra info in the following sentence 


vk$dpg <- as.numeric(str_detect(vk$origin2, "(dorp)|(plaats)|(gemeente)|(graafschap)|(stad)|(deel)|(Friesland)")) # origin info too regional 
vk <- vk %>% mutate(origin2 = ifelse((dpg==1), NA, origin2)) # removing regional origin info
vk <- subset(vk, select = -dpg) # removing intermediate variable

# Sometimes, there were multiple countries mentioned. Take only the first:
vk$origin2 <- str_remove(vk$origin2, "\\,.*") # Only first
vk$origin2 <- str_remove(vk$origin2, "\\s(en).*") # Only first 
vk$origin2 <- str_remove(vk$origin2, "\\s(of).*") # Only first 


# Origin3: pretty neat
vk$origin3 <- str_remove(vk$origin3, "D(i)?e(ze)? (familie)?(achter)?(beroeps)?naam") # slipped through the regex
vk$origin3 <- str_remove(vk$origin3, "Een (familie)?(achter)?(beroeps)?naam") # slipped through the regex
vk$origin3 <- str_remove(vk$origin3, "Zijn (familie)?(achter)?(beroeps)?naam") # slipped through the regex
vk$origin3 <- str_remove(vk$origin3, "Als (familie)?(achter)?(beroeps)?naam") # slipped through the regex
vk$origin3 <- str_remove(vk$origin3, "Joodse (familie)?(achter)?naam")
vk$origin3 <- str_remove(vk$origin3, "Bijbelse (familie)?(achter)?naam")


# Setting empty origin variables to NA (Dutch or unfound foreign)
vk <- vk %>%
  mutate(origin1 = as.character(ifelse(origin1==""|origin1=="character(0)", NA, origin1)),
         origin2 = as.character(ifelse(origin2==""|origin2=="character(0)", NA, origin2)),
         origin3 = as.character(ifelse(origin3==""|origin3=="character(0)", NA, origin3)))



# Finally, the tag "andere taal" was used to distinguish foreign names of unknown origin from known Dutch names. 
vk <- vk %>%
  mutate(origin4 = ifelse((as.numeric(str_detect(kenmerken, "andere taal"))==1), "non-Dutch", NA))

voorvoegselsnl <- c("'t", "d'", "de", "de la", "den", "del", "der", "des", "in 't", "op de", "op den", "ten", "ter", "tes", "van", "van 't", "van de" , "van der", "van den") 

vk$nl <- 0

for (i in 1: length(vk$lastname_df.lastname)) {
  if (sum(str_detect(lastname_df$np[i], voorvoegselsnl))>0) {
    vk$nl[i] <- 1
  }
}

vk$dutch <- ifelse(((vk$no_info==1 | vk$no_info=="non_Dutch") & !vk$nl==1), 0, 1)
names(vk)
vk %>% select(c("lastname_df.id", "lastname_df.lastname", "dutch"))

save(vk, file="vk20221006.RData") 

names_df$dutch <- vk$dutch

save(names_df, file="names_df_v20221006.RData") 

Ik zou alles waar no_info op 1 staat of waar origin4 op “non-Dutch” staat coderen als buitenlands!