# models.py
import numpy as np
class LanguageModel(object):
def get_next_char_log_probs(self, context) -> np.ndarray:
Returns a log probability distribution over the next characters given a context.
The log should be base e
NOTE: You should make sure you call model.eval() to determinize inference here (turns off dropout
layers in TransformerEncoder).
:param context: the string context that the LM conditions on
:return: A numpy vector log P(y | context) where y ranges over the output vocabulary.
raise Exception(“Only implemented in subclasses”)
def get_log_prob_sequence(self, next_chars, context) -> float:
Scores a bunch of characters following context. That is, returns
log P(nc1, nc2, nc3, … | context) = log P(nc1 | context) + log P(nc2 | context, nc1), …
The log should be base e
NOTE: You should make sure you call model.eval() to determinize inference here (turns off dropout
layers in TransformerEncoder).
:param next_chars:
:param context:
:return: The float probability
raise Exception(“Only implemented in subclasses”)
class UniformLanguageModel(LanguageModel):
def __init__(self, voc_size):
self.voc_size = voc_size
def get_next_char_log_probs(self, context):
return np.ones([self.voc_size]) * np.log(1.0/self.voc_size)
def get_log_prob_sequence(self, next_chars, context):
return np.log(1.0/self.voc_size) * len(next_chars)
class NeuralLanguageModel(LanguageModel):
def __init__(self):
raise Exception(“Implement me”)
def get_next_char_log_probs(self, context):
raise Exception(“Implement me”)
def get_log_prob_sequence(self, next_chars, context):
raise Exception(“Implement me”)
def train_lm(args, train_text, dev_text, vocab_index):
:param args: command-line args, passed through here for your convenience
:param train_text: train text as a sequence of characters
:param dev_text: dev text as a sequence of characters
:param vocab_index: an Indexer of the character vocabulary (27 characters)
:return: a NeuralLanguageModel instance trained on the given data
raise Exception(“Implement me”)