Python: Remove Duplicates from Text File
Question: I am new to python. I want to remove duplicate word

and except English word i want to delete all other word and blank line.

purely English word only i want to extract.

i have some text file which contain such like following

    
    
    aaa
    bbb
    aaa223
    
    aaa
    ccc
    ddd
    
    kei60:
    sj@6999
    
    
    jack02
    jparkj
    

so after process duplicate i want to get result following

    
    
    aaa
    bbb
    ccc
    ddd
    jparkj
    

following is what i tried script source.

if anyone help me much appreciate! thanks!

    
    
    # read a text file, replace multiple words specified in a dictionary
    # write the modified text back to a file
    
    import re
    
    def replace_words(text, word_dic):
        """
        take a text and replace words that match a key in a dictionary with
        the associated value, return the changed text
        """
        rc = re.compile('|'.join(map(re.escape, word_dic)))
        def translate(match):
            return word_dic[match.group(0)]
        return rc.sub(translate, text)
    
    
    
    def main():
        test_file = "prxtest.txt"
        # read the file
        fin = open(test_file, "r")
        str2 = fin.read()
        fin.close()
    
        # the dictionary has target_word:replacement_word pairs
        word_dic = {
        '.': '\n',
        '"': '\n',
        '<': '\n',
        '>': '\n',
        '!': '\n',
        "'": '\n',
        '(': '\n',
        ')': '\n',
        '[': '\n',
        ']': '\n',
        '@': '\n',
        '#': '\n',
        '$': '\n',
        '%': '\n',
        '^': '\n',
        "&": '\n',
        '*': '\n',
        '_': '\n',
        '+': '\n',
        '-': '\n',
        '=': '\n',
        '}': '\n',
        '{': '\n',
        '"': '\n',
        ";": '\n',
        ':': '\n',
        '?': '\n',
        ',': '\n',
        '`': '\n',
        '~': '\n',
        '1': '\n',
        '2': '\n',
        '3': '\n',
        '4': '\n',
        "5": '\n',
        '6': '\n',
        '7': '\n',
        '8': '\n',
        '9': '\n',
        '0': '\n',
        ' ': '\n'}
    
        # call the function and get the changed text
        str3 = replace_words(str2, word_dic)
    
        # write changed text back out
        fout = open("clean.txt", "w")
        fout.write(str3)
        fout.close()
    
    if __name__ == "__main__":
    
        main()
    


Answer: This will capture lines containing only letters:

    
    
    fin = open(test_file, 'r')
    fout = open('clean.txt', 'w')
    
    s = set()
    for line in fin:
        if line.rstrip().isalpha():
            if not line in s:
                s.add(line)
                fout.write(line)
    
    fin.close()
    fout.close()
    

