Creando ARFF a partir de frecuencias de palabras

Question

Tengo un código que me da una lista de palabras con sus frecuencias que ocurren en el texto, estoy buscando hacerlo para que el código convierta automáticamente las primeras 10 palabras en un ARFF con

wordfrequencies @RELATION

@ATTRIBUTE cadena de palabras de frecuencia @ATTRIBUTE numérico

y el top 10 como datos con su frecuencia.

estoy luchando con la forma de hacer esto con mi código actual

import re 
import nltk 

# Quran subset 
filename = 'subsetQuran.txt' 

# create list of lower case words 
word_list = re.split('\s+', file(filename).read().lower()) 
print 'Words in text:', len(word_list) 

word_list2 = [w.strip() for w in word_list if w.strip() not in nltk.corpus.stopwords.words('english')] 



# create dictionary of word:frequency pairs 
freq_dic = {} 
# punctuation and numbers to be removed 
punctuation = re.compile(r'[-.?!,":;()|0-9]') 
for word in word_list2: 
    # remove punctuation marks 
    word = punctuation.sub("", word) 
    # form dictionary 
    try: 
     freq_dic[word] += 1 
    except: 
     freq_dic[word] = 1 


print '-'*30 

print "sorted by highest frequency first:" 
# create list of (val, key) tuple pairs 
freq_list2 = [(val, key) for key, val in freq_dic.items()] 
# sort by val or frequency 
freq_list2.sort(reverse=True) 
freq_list3 = list(freq_list2) 
# display result 
for freq, word in freq_list2: 
    print word, freq 
f = open("wordfreq.txt", "w") 
f.write(str(freq_list3)) 
f.close() 

Cualquier ayuda con esto se aprecia, una forma de hacer esto es realmente devanando los sesos!

Answer 1

espero que no le importe la ligera reescritura:

import re 
import nltk 
from collections import defaultdict 

# Quran subset 
filename = 'subsetQuran.txt' 

# create list of lower case words 
word_list = open(filename).read().lower().split() 
print 'Words in text:', len(word_list) 

# remove stopwords 
word_list = [w for w in word_list if w not in nltk.corpus.stopwords.words('english')] 

# create dictionary of word:frequency pairs 
freq_dic = defaultdict(int) 

# punctuation and numbers to be removed 
punctuation = re.compile(r'[-.?!,":;()|0-9]') 
for word in word_list: 
    # remove punctuation marks 
    word = punctuation.sub("", word) 
    # increment count for word 
    freq_dic[word] += 1 

print '-' * 30 

print "sorted by highest frequency first:" 
# create list of (frequency, word) tuple pairs 
freq_list = [(freq, word) for word, freq in freq_dic.items()] 

# sort by descending frequency 
freq_list.sort(reverse=True) 

# display result 
for freq, word in freq_list: 
    print word, freq 

# write ARFF file for 10 most common words 
f = open("wordfreq.txt", "w") 
f.write("@RELATION wordfrequencies\n") 
f.write("@ATTRIBUTE word string\n") 
f.write("@ATTRIBUTE frequency numeric\n") 
f.write("@DATA\n") 
for freq, word in freq_list[ : 10]: 
    f.write("'%s',%d\n" % (word, freq)) 
f.close()