human
I am working on the file "Demo.ipynb".
The first task is:
Import NLTK to seperate a document into sentences and import Gensim to actually train the Word2Vec models.
Can you write Python program for it?
 ... Can you provide a the Python program you requested?
assistant
#Here is the Python program you requested.

pattern = re.compile(r'[\W\d]')

try:
    nltk.data.find("tokenizers/punkt")
except LookupError:
    punkt = nltk.download('punkt')

def init_tokenizer(lang):
    model = 'tokenizers/punkt/{}.pickle'.format(lang.lower())
    return nltk.data.load(model)

def sentences(doc):
    tokenizer = init_tokenizer("german")
    sent = tokenizer.tokenize(doc.strip())
    return sent

def token(sentence):
    letters = pattern.sub(" ", sentence)
    words = letters.lower().split()
    words = [word for word in words if len(word) > 1]
    return words

class FileCorpus:
    def __init__(self, files, encoding='UTF-8'):
        self.files = files
        self.encoding = encoding

    def __iter__(self):
        for file in self.files:
            for doc in open(file, encoding=self.encoding):
                yield doc
    
    def sentences(self):
        documents = [sentences(document) for document in self]
        t = [token(sentence) for document in documents for sentence in document]
        return t

def build_model(name, coll):
    sentences = FileCorpus(coll).sentences()
    shuffle(sentences)
    model = Word2Vec(sentences, workers=4, iter=10, size=100, window=2, sg=1, hs=1)
    model.save(path.join('models','{}.w2v'.format(name)))
    return model

npd = build_model('NPD', [path.join('data', file) 
                          for file in ['NPD.txt', 'NPD_MV.txt', 'NPD_Sachsen.txt']])
spd = build_model('SPD', [path.join('data', file) 
                          for file in ['SPD_Inland.txt', 'SPD_International.txt', 'SPD_Parteileben.txt']])
cdu = build_model('CDU', [path.join('data', file) 
                          for file in ['CDU.txt', 'CDU_EU.txt', 'CDU_Fraktion.txt']])
fdp = build_model('FDP', [path.join('data', file) 
                          for file in ['FDP.txt', 'FDP_Fraktion.txt']])
grüne = build_model('GRÜNE', [path.join('data', file) 
                          for file in ['Grüne.txt', 'Grüne_Fraktion.txt']])
linke = build_model('LINKE', [path.join('data', file) 
                          for file...