You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
129 lines
4.5 KiB
129 lines
4.5 KiB
import os, json
|
|
from flask import Markup
|
|
|
|
from nltk import tokenize, pos_tag, RegexpTokenizer
|
|
tokenizer = RegexpTokenizer(r'\w+') # initialize tokenizer
|
|
|
|
import pprint
|
|
pp = pprint.PrettyPrinter(indent=4)
|
|
|
|
def load_index():
|
|
if os.path.isfile('index.json') == False:
|
|
tfidf.create_index()
|
|
f = open('index.json').read()
|
|
index = json.loads(f)
|
|
return index
|
|
|
|
def get_random(x, y):
|
|
from random import randint
|
|
return randint(x, y)
|
|
|
|
def generate_random_rgb():
|
|
r = get_random(0, 255)
|
|
g = get_random(0, 255)
|
|
b = get_random(0, 255)
|
|
return r, g, b
|
|
|
|
def get_pos():
|
|
# ---
|
|
# Note: NLTK provides documentation for each tag,
|
|
# which can be queried using the tag, e.g.
|
|
# nltk.help.upenn_tagset('RB'), or a regular expression,
|
|
# e.g. nltk.help.upenn_tagset('NN.*'). Some corpora
|
|
# have README files with tagset documentation,
|
|
# see nltk.corpus.???.readme(), substituting in the name
|
|
# of the corpus. -- http://www.nltk.org/book/ch05.html
|
|
# ---
|
|
|
|
# data {
|
|
# 'word' : {
|
|
# 'count' : 8,
|
|
# 'sentences' : {
|
|
# 'filename' : [
|
|
# 'This is a sentence.',
|
|
# 'This is another sentence.'
|
|
# ]
|
|
# }
|
|
# }
|
|
# }
|
|
|
|
index = load_index()
|
|
sentences_all = [index[document]['sentences'] for document, _ in index.items()]
|
|
data = {}
|
|
data['ADJ'] = {}
|
|
data['PRE'] = {}
|
|
filenames = [filename for filename, _ in index.items()]
|
|
# print(filenames)
|
|
for i, sentences in enumerate(sentences_all):
|
|
r, g, b = generate_random_rgb()
|
|
for sentence in sentences:
|
|
pos = pos_tag(tokenizer.tokenize(sentence))
|
|
# print(pos)
|
|
for word, tag in pos:
|
|
if 'JJ' in tag:
|
|
# ---
|
|
# JJ: adjective or numeral, ordinal
|
|
# For example: third ill-mannered pre-war regrettable oiled calamitous first separable
|
|
# ectoplasmic battery-powered participatory fourth still-to-be-named
|
|
# multilingual multi-disciplinary ...
|
|
if word.lower() not in data['ADJ']:
|
|
data['ADJ'][word.lower()] = {}
|
|
if 'sentences' not in data['ADJ'][word.lower()]:
|
|
data['ADJ'][word.lower()]['sentences'] = {}
|
|
if filenames[i] not in data['ADJ'][word.lower()].keys():
|
|
data['ADJ'][word.lower()]['sentences'][filenames[i]] = []
|
|
s = Markup(sentence.replace(word, '<strong class="query" style="color:rgba({r},{g},{b},1); background-image: radial-gradient(ellipse, rgba({r},{g},{b},0.4), rgba({r},{g},{b},0.2), transparent, transparent);">{word}</strong>'.format(r=r, b=b, g=g, word=word)))
|
|
if s not in data['ADJ'][word.lower()]['sentences'][filenames[i]]:
|
|
data['ADJ'][word.lower()]['sentences'][filenames[i]].append(s)
|
|
if 'TO' in tag or 'IN' in tag:
|
|
# ---
|
|
# TO: "to" as preposition (voorzetsel) or infinitive marker (oneindige beïnvloeder?)
|
|
# For example: to
|
|
# ---
|
|
# IN: preposition or conjunction (voegwoord, samenstelling, verbinding), subordinating (ondergeschikt)
|
|
# For example: astride among uppon whether out inside pro despite on by throughout
|
|
# below within for towards near behind atop around if like until below
|
|
# next into if beside ...
|
|
# ---
|
|
if word.lower() not in data['PRE']:
|
|
data['PRE'][word.lower()] = {}
|
|
if 'sentences' not in data['PRE'][word.lower()]:
|
|
data['PRE'][word.lower()]['sentences'] = {}
|
|
if filenames[i] not in data['PRE'][word.lower()]['sentences'].keys():
|
|
data['PRE'][word.lower()]['sentences'][filenames[i]] = []
|
|
s = Markup(sentence.replace(word, '<strong class="query" style="color:rgba({r},{g},{b},1); background-image: radial-gradient(ellipse, rgba({r},{g},{b},0.4), rgba({r},{g},{b},0.2), transparent, transparent);">{word}</strong>'.format(r=r, b=b, g=g, word=word)))
|
|
if s not in data['PRE'][word.lower()]['sentences'][filenames[i]]:
|
|
data['PRE'][word.lower()]['sentences'][filenames[i]].append(s)
|
|
|
|
# count number of results for each word
|
|
for word_type, words in data.items():
|
|
for word, _ in words.items():
|
|
# print(filenames)
|
|
count = 0
|
|
for filename, sentences in data[word_type][word]['sentences'].items():
|
|
# print(filename)
|
|
count += len(sentences)
|
|
data[word_type][word.lower()]['count'] = count
|
|
count = 0
|
|
|
|
data_sorted = {}
|
|
for word_type, words in data.items():
|
|
tmp = []
|
|
for word, _ in words.items():
|
|
count = data[word_type][word.lower()]['count']
|
|
tmp.append([count, word])
|
|
i = 0
|
|
tmp.sort(reverse=True)
|
|
print('tmp', tmp)
|
|
for count, word in tmp:
|
|
if word_type not in data_sorted:
|
|
data_sorted[word_type] = {}
|
|
data_sorted[word_type][i] = {}
|
|
data_sorted[word_type][i][word.lower()] = data[word_type][word.lower()]
|
|
i += 1
|
|
|
|
print(data_sorted)
|
|
return data_sorted, index
|
|
|
|
# data, index = get_pos()
|
|
# pp.pprint(data)
|