Here, I summarized the main point of my master thesis and Github .
In this research project, I was looking for Natural Language Processing (NLP) techniques to improve the quality of geodata source retrieval in Spatial Data infrastructures (SDIs) using semantic keywords for the geographic phenomena requested. For example, a user wants to answer this spatail question: "How much is the concentration of bold eagels in Holland National Parks?". In this question, "parks" and "eagles" are geographic phenomena and if users search to find relevant datasets in a Dutch national geospatial information brokers such as PDOK or National geo-registry, the results of their search might be empty. Simply because the keywords recorded in metadata are "parken" and "dier (animal)" or "vogels (birds)".
To address this problem, I used English WordNet and Google translate API to help different users with language limitations and specific domains by capturing the semantic and linguistic content in metadata. Therefore, when a query executes in English, the algorithm reformulates and enriches queries based on the synonyms, hyponym, and hypernym gathered from the English WordNet and the results of the semantic keywords is tranlsted to Dutch and matched with metadata. This affects the recall and precision of geodata retrieval. This approach causes improvement in the precision and recall of geo-datasets by 1% and 22% respectively.
For query expansion, I developed python codes as shown in the below. Query expansion in WordNet is carried out in five phases. The first phase represents hierarchal relations and computing synsets (i.e., synonyms, hypernyms, and hyponyms) of keywords. The second and third phases are computing the similarity and semantic overlay, respectively. Next, the query expansion results are translated into Dutch. Finally, the SPARQL query is executed against RDF metadata, and keywords are matched with metadata.
import nltk
from nltk.corpus import wordnet
from google_trans_new import google_translator
import rdflib
from fuzzywuzzy import fuzz, process
import re
import csv
from pattern.en import pluralize, singularize
from timeit import default_timer as timer
from datetime import timedelta
start = timer()
#### setting inputs
Path_input = "C:\\0000Pythoncodes\\API\\final\\input\\"
#### input user keyword
with open(Path_input + "KeywordsEnglish.txt") as User_keyword_English:
User_keyword = User_keyword_English.read()
User_keyword = User_keyword.replace(" ", "_")
name = User_keyword.split('\n')
#### input keywords extracted from the RDF metadata
with open(Path_input + "Data_answer_first.csv") as English:
metadataEnglish = English.read()
key_list_Meta = metadataEnglish.split('\n')
#### setting outputs
path_output = "C:\\0000Pythoncodes\\API\\final\\output\\WordNet\\"
#### this function is used for providing output for google translate and results of query expansion
def output(api):
path_output = "C:\\0000Pythoncodes\\API\\final\\output\\WordNet\\"
with open(path_output + User_keyword + ".csv", "w", newline='') as w:
wr = csv.writer(w, quoting=csv.QUOTE_ALL)
for word in api:
if not word.strip(): continue
wr.writerow([word])
#### output similarity links
def outputsimilarity(similarity,path_output):
with open(path_output + User_keyword + ".csv", "w", newline='') as w:
wr = csv.writer(w, quoting=csv.QUOTE_ALL)
for word in similarity:
if not word.strip(): continue
wr.writerow([word])
def outputlinks(number, name, mylist):
with open(path_output + name + number + ".csv", "w", newline='') as w:
wr = csv.writer(w, quoting=csv.QUOTE_ALL)
for word in mylist:
wr.writerow([word])
#### this functtion returns keywords to the root
def lemmatize_keywords(words):
lemmatizer = nltk.WordNetLemmatizer()
lemmatized_keyword = (lemmatizer.lemmatize(words))
return lemmatized_keyword
#### synonyms_synsets() function calculate synonyms and lemmas for the user keyword
setAList = [lemmatize_keywords(User_keyword)]
def synonyms_synsets(AList):
synonyms = []
for TheList in AList:
for syn in wordnet.synsets(TheList):
for l in syn.lemmas():
synStr = l.name()
SynStrlemma = lemmatize_keywords(synStr) # return synonyms to their roots
SynStrlemmaLower = SynStrlemma.lower()
synonyms.append(SynStrlemmaLower)
semanticKeywords = list(synonyms) # List of synonyms
semanticKeywordsUnique = list(dict.fromkeys(semanticKeywords))[0:4] # remove duplicates from a List
return semanticKeywordsUnique
#### hyponyms() function returns the list hyponyms for user inputs
def hyponyms(AList):
for root in AList:
hypoHyper = wordnet.synset(root + '.n.01')
hypo_hyponyms = lambda s: s.hyponyms()
list(hypoHyper.closure(hypo_hyponyms)) == hypoHyper.hyponyms()
hyo_hypo_hyponyms_list = list(hypoHyper.closure(hypo_hyponyms))
#### compute hyponyms
hyponyms_list = []
if len(hyo_hypo_hyponyms_list) > 0:
for hyponym in hyo_hypo_hyponyms_list:
hyponymStr = hyponym.name()
hyponymStrSpl = hyponymStr.split('.')[0]
hyponyms_list.append(hyponymStrSpl)
hyponyms_list_final = list(hyponyms_list)
else:
hyponyms_list_final = ['none'] # if the list is empty
return hyponyms_list_final
#### hypernyms() function returns the list hypernyms for user inputs
def hypernyms(AList):
for root in AList:
hypoHyper = wordnet.synset(root + '.n.01')
hyper_hypernyms = lambda s: s.hypernyms()
list(hypoHyper.closure(hyper_hypernyms)) == hypoHyper.hypernyms()
hye_hyper_hypernyms_list = list(hypoHyper.closure(hyper_hypernyms))
hypernyms_list = []
if len(hye_hyper_hypernyms_list) > 0:
for hyp in hye_hyper_hypernyms_list:
hyeStr = hyp.name()
hyeStrSpl = hyeStr.split('.')[0]
hypernyms_list.append(hyeStrSpl)
hypernyms_list_final = list(hypernyms_list)
else:
hypernyms_list_final = ['none']
return hypernyms_list_final
### Set A
Final_list_synsets_SetA = synonyms_synsets(setAList)
Final_list_synsets_SetA.extend(hyponyms(setAList))
Final_list_synsets_SetA.extend(hypernyms(setAList))
"""the below function compute relatedness"""
####this function compute set B and the output is the result of A ∩ B
def metadata_synset_lists(Meta, synsets_SetA):
synonyms_semantic_final_AB = []
final = []
for TheList in Meta:
synonyms_meta = []
for syn in wordnet.synsets(TheList):
for l in syn.lemmas():
synStr_meta = l.name()
SynStrlemma_meta = lemmatize_keywords(synStr_meta)
SynStrlemmaLower_meta = SynStrlemma_meta.lower()
synonyms_meta.append(SynStrlemmaLower_meta)
semanticKeywords = list(synonyms_meta)
semanticKeywordsUnique_meta_setB = list(dict.fromkeys(semanticKeywords))[0:4]
intersection_keyword_AB = list(set.intersection(set(synsets_SetA), set(semanticKeywordsUnique_meta_setB)))
if len(intersection_keyword_AB) >= 1:
for intersectionAB in intersection_keyword_AB:
if len(intersectionAB) > 2:
synonyms_semantic_final_AB.append(intersectionAB)
final.append(synonyms_semantic_final_AB)
flat_list = [item for sublist in final for item in sublist]
synonyms_semantic_final_meta_setAandB = list(dict.fromkeys(flat_list))
return synonyms_semantic_final_meta_setAandB
"""this is smilarity function"""
### set C
#### the similarity_score function checks distance between the user keyword with metadata keywords
def similarity_score(keyword, Meta):
lookup_keyword = wordnet.synset(lemmatize_keywords(keyword) + '.n.01')
syn_answer_list = []
answer_scores_list = []
score_similarity = []
for syn_answer in Meta:
syn_answer = lemmatize_keywords(syn_answer)
lookup_meta = wordnet.synset(syn_answer + '.n.01')
syn_scores = str(round(lookup_keyword.wup_similarity(lookup_meta), 2))
syn_answer_split = syn_answer.split(',')
syn_answer_list.append(syn_answer_split)
answer_scores_split = syn_scores.split(',')
answer_scores_list.append(answer_scores_split)
concatenate_list = [x + y for x, y in zip(syn_answer_list, answer_scores_list)] # Making a list by concatenating items in parallel
for con in concatenate_list:
itemTofloat = float(con[1])
if itemTofloat > 0.69:
concatenate_list_strings_str = con[0]
score_similarity.append(concatenate_list_strings_str)
concatenate_list_strings = score_similarity
return(concatenate_list_strings)
setAandB = metadata_synset_lists(key_list_Meta, Final_list_synsets_SetA)
setC = similarity_score(User_keyword, key_list_Meta)[0:3]
setAandB.extend(setC)
setAandB = setAandB
for AB in setAandB :
setAandBandC_p = pluralize(AB)
setAandBandC_p= setAandBandC_p.split('\n')
setAandB.extend(setAandBandC_p)
setAandB.extend(name)
setAandBandC = setAandB
setAandBandC_final = list(dict.fromkeys(setAandBandC))
print("semantic keywords in English:", setAandBandC_final)
key_list = []
for name in setAandBandC_final:
name = name.replace("_", " ")
translator = google_translator()
translate_text = translator.translate(name,lang_tgt='nl')[0:-1]
key_list.append(translate_text)
key = key_list
print("semantic keywords in Dutch:", key)
g = rdflib.Graph()
RDF = g.parse(Path_input + "metadata_pdok_all_removed.ttl", format="ttl")
##### Run a Query
lineList = []
for k in key:
k = k.replace("-","")
qres = RDF.query(""" SELECT ?s ?k WHERE {
?s ||| ?k }""")
#### start extracting URIs and metadata content for keyword matched with correspond URI
#### remove unwanted characters and strings
for row in qres:
strrow = (str(row)).replace("rdflib.term.URIRef", "")
metadata = strrow.replace(")", "")
#### keyword matching and scoring the result of matched keywords
Ratio = fuzz.partial_ratio(k.lower(), metadata.lower())
##### extract URIs with 80% similarity by string pattern matching to allow special characters to be used without invoking their special meaning.
if Ratio == 100: ##80 for prural
link_regex = re.compile("((https?):((//)|(\\\\))+([\w\d:#@%/;$()~_?\+-=\\\.&](#!)?)*)")
links = re.findall(link_regex, metadata)
##### filter out unwanted characters from URLs using index
for word in links:
word = str(word).split(",")[0]
URIs = word.replace("('", "")
##### filter out unwanted URLs in description from URLs linked to metadata
if len(URIs) > 50:
if (".pdf" in URIs):
pdf = URIs
elif ("kadaster.nl" in URIs):
kadaster = URIs
elif ("cbs.nl" in URIs):
cbs = URIs
else:
lineList.append(URIs)
mylist = sorted(list(dict.fromkeys(lineList)))
# result output
count = []
count1 = []
for uri in mylist[:105]:
if len(uri) > 100:
result_search1 = uri
count1.append(result_search1)
print(uri)
else:
result_search = "https://data.labs.kadaster.nl/pdok/metadata/browser?resource=" + uri
print(result_search)
count.append(result_search)
count1.extend(count)
number = (len(mylist))
number = str(number)
print("this is number of links:", number)
#### write a URI to a file a csv file
with open(path_output + name + "_" + k + number + ".csv", "w", newline='') as w:
wr = csv.writer(w, quoting=csv.QUOTE_ALL)
for word in mylist:
wr.writerow([word])
end = timer()
print(timedelta(seconds=end-start))