#Step 1
f = open('stopwords.txt', "r")
content = f.readlines()
stopwords = [x.replace('\n','') for x in content]
print("List of Stop words")
print(stopwords)
#Step 3: read from the text file, line by line and clean words
allWords = []
fl = open('news.txt', "r", errors='ignore')
lines = fl.readlines()
print("\n\n")
f2 = open('news.txt', "r", errors='ignore')
print(f2.read())
for line in lines:
      #remove punctuations
      punctuations = ["?",".","!","/",";",":","-","(",")",",","$",'"',".",'\n']
      for punctuation in punctuations:
            if punctuation in line:
                  line = line.replace(punctuation, "")
      #change to lowercase
      line = line.lower()
      words = line.split(' ')
      #add words to a wordlist
      allWords.extend(words)
print('\nWordlist: ')
print(allWords)
allWords.sort()
print('\nWord List Sorted: ')
print(allWords)
#remove stopwords, & numbers
for w in allWords:
      for _ in range(allWords.count(w)):
            allWords.remove(w)
print('\nStop Word Removed ')
print(allWords)  
nonNumeric = []  
for w in allWords:
      if (w.isnumeric()==False):
            nonNumeric.append(w)
print('\nNumbers Removed ')
print(nonNumeric)
#remove duplicates
nonDuplicates = []
for w in nonNumeric:
      if w not in nonDuplicates:
            nonDuplicates.append(w)
print('\nDuplicated words Removed ')     
print(nonDuplicates)