question:¿Qué le falta a este código para obtener una predicción y no <pad> <pad>?: import torch import torch.nn as nn import time from torchtext.datasets import PennTreebank from torchtext.data.functional import to_map_style_dataset from torchtext.data.utils import get_tokenizer from torchtext.vocab import build_vocab_from_iterator from torch.utils.data import DataLoader # 1. Define the model class TransformerModel(nn.Module): def __init__(self, vocab_size, d_model, nhead, num_layers, dropout=0.1): super(TransformerModel, self).__init__() self.embedding = nn.Embedding(vocab_size, d_model) self.transformer = nn.Transformer(d_model, nhead, num_layers, dropout=dropout) self.fc = nn.Linear(d_model, vocab_size) def forward(self, src, tgt): src = self.embedding(src) tgt = self.embedding(tgt) x = self.transformer(src, tgt) x = self.fc(x) return x # 2. Preparar datos train_data_raw = to_map_style_dataset(PennTreebank(split='train')) tokenizer = get_tokenizer("spacy", "en_core_web_sm") # Agregar tokens especiales (UNK y PAD) al vocabulario specials = ['<unk>','<pad>'] vocab = build_vocab_from_iterator((tokenizer(y) for y in train_data_raw), specials=specials) vocab.set_default_index(vocab['<unk>']) # Dividir los datos en conjuntos de entrenamiento y validación train_ratio = 0.8 train_data_size = int(len(train_data_raw) * train_ratio) train_data = train_data_raw[:train_data_size] valid_data = train_data_raw[train_data_size:] # 3. Entrenar el modelo def generate_pairs(tokens, shift=1): source = [tokens[i] for i in range(0, len(tokens) - shift)] target = [tokens[i] for i in range(shift, len(tokens))] return source, target def tokenize_data(text, max_seq_len=None): tokens = tokenizer(text)[:max_seq_len] return torch.tensor([vocab[token] if token in vocab else vocab['<unk>'] for token in tokens], dtype=torch.long) def create_batches(data, batch_size, max_seq_len=None): batched_data = [] for i in range(0, len(data), batch_size): batched_text = [] for text in data[i:i + batch_size]: tokens = tokenize_data(text, max_seq_len=max_seq_len) if len(tokens) <= 1: continue batched_text.append(tokens) batched_data.append(batched_text) return batched_data def collate_fn(batch): batch_srcs = [] batch_tgts = [] max_len = max([len(text) for text in batch]) for text in batch: source, target = generate_pairs(text) source = torch.tensor(source, dtype=torch.long) target = torch.tensor(target, dtype=torch.long) source = torch.cat([source, torch.full((max_len - len(source),), vocab['<pad>'], dtype=torch.long)], dim=0) target = torch.cat([target, torch.full((max_len - len(target),), vocab['<pad>'], dtype=torch.long)], dim=0) batch_srcs.append(source) batch_tgts.append(target) return torch.stack(batch_srcs).T, torch.stack(batch_tgts).T vocab_size = len(vocab) d_model = 768 nhead = 8 # Ajuste de hparámetro num_layers = 6 # Ajuste de hiperparámetro num_epochs = 30 # Increase the number of epochs learning_rate = 1e-3 # Experiment with different learning rates optimizer_choice = "Adam" # Choose the optimizer: "Adam", "RMSprop", or "Adagrad" batch_size = 64 # Ajuste de hiperparámetro max_seq_len = 50 # Limitar la longitud máxima de la secuencia para evitar problemas de memoria device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_batches = create_batches(train_data, batch_size, max_seq_len=max_seq_len) valid_batches = create_batches(valid_data, batch_size, max_seq_len=max_seq_len) train_loader = DataLoader(train_batches, batch_size=None, collate_fn=collate_fn) valid_loader = DataLoader(valid_batches, batch_size=None, collate_fn=collate_fn) model = TransformerModel(vocab_size=vocab_size, d_model=d_model, nhead=nhead, num_layers=num_layers, dropout=0.1).to(device) loss_function = nn.CrossEntropyLoss() # Use different optimizers if optimizer_choice == "Adam": optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) elif optimizer_choice == "RMSprop": optimizer = torch.optim.RMSprop(model.parameters(), lr=learning_rate) elif optimizer_choice == "Adagrad": optimizer = torch.optim.Adagrad(model.parameters(), lr=learning_rate) else: raise ValueError("Invalid optimizer choice") def train(model, epoch): model.train() train_loss = 0 for i, (source, target) in enumerate(train_loader): source = source.to(device) target = target.to(device) optimizer.zero_grad() output = model(source, target[:-1]) loss = loss_function(output.reshape(-1, vocab_size), target[1:].reshape(-1)) loss.backward() optimizer.step() train_loss += loss.item() if i % 1000 == 0: print(f'Epoch: {epoch + 1},tLoss: {loss.item()}') return train_loss / len(train_loader) for epoch in range(num_epochs): start_time = time.time() train_loss = train(model, epoch) end_time = time.time() elapsed_time = end_time - start_time print(f"Epoch: {epoch + 1},tAverage Loss: {train_loss},tTime taken: {elapsed_time} seconds") # 4. Evaluar el modelo en el conjunto de validación def evaluate(model, valid_loader): model.eval() valid_loss = 0 with torch.no_grad(): for source, target in valid_loader: source = source.to(device) target = target.to(device) output = model(source, target[:-1]) loss = loss_function(output.reshape(-1, vocab_size), target[1:].reshape(-1)) valid_loss += loss.item() return valid_loss / len(valid_loader) # 5. Entrenar y evaluar el modelo en cada época for epoch in range(num_epochs): start_time = time.time() train_loss = train(model, epoch) valid_loss = evaluate(model, valid_loader) end_time = time.time() elapsed_time = end_time - start_time print(f"Epoch: {epoch + 1},tAverage Train Loss: {train_loss},tAverage Valid Loss: {valid_loss},tTime taken: {elapsed_time} seconds") # 6. Guardar el modelo entrenado torch.save(model.state_dict(), "trained_model.pth") # 7. Cargar el modelo def load_model(path, device): loaded_model = TransformerModel(vocab_size=vocab_size, d_model=d_model, nhead=nhead, num_layers=num_layers).to(device) loaded_model.load_state_dict(torch.load(path)) loaded_model.eval() return loaded_model loaded_model = load_model("trained_model.pth", device) # 8. Predecir texto def decode_output(output): probabilities = torch.softmax(output, dim=2) predicted_indices = torch.argmax(probabilities, dim=2) return " ".join(vocab.lookup_tokens(predicted_indices.cpu().numpy().squeeze().tolist())) def predict_next_token(input_text, model): tokens = tokenize_data(input_text) source, target = generate_pairs(tokens, shift=1) source = torch.tensor(source, dtype=torch.long).unsqueeze(1).to(device) target = torch.tensor(target, dtype=torch.long).unsqueeze(1).to(device) with torch.no_grad(): output = model(source, target[:-1]) return decode_output(output) input_text = "My favorite color is" predicted_output = predict_next_token(input_text, loaded_model) print(f"Predicted output: {predicted_output}")

question:¿Cuál es un conjunto de datos de texto mejor que PeenTreebank para entren el modelo que te mostré?

question:Por favor, configúrame el código para usar Wikitext y ajústame los siguientes parámetros para que genere una buena predicción, así se demore el proceso de entrenamiento: vocab_size = len(vocab) d_model = 768 nhead = 8 # Ajuste de hparámetro num_layers = 6 # Ajuste de hiperparámetro num_epochs = 30 # Increase the number of epochs learning_rate = 1e-3 # Experiment with different learning rates optimizer_choice = “Adam” # Choose the optimizer: “Adam”, “RMSprop”, or “Adagrad” batch_size = 64 # Ajuste de hiperparámetro max_seq_len = 50 # Limitar la longitud máxima de la secuencia para evitar problemas de memoria

question:Por favor, dame una buena configuración en los hiperparámetros para evitar esto: Epoch: 1, Loss: 9.567360877990723 Epoch: 1, Average Loss: 3.7003726646474107, Time taken: 292.0087857246399 seconds Epoch: 2, Loss: 3.559448480606079 Epoch: 2, Average Loss: 3.618738709293844, Time taken: 292.34765434265137 seconds Epoch: 3, Loss: 3.5392792224884033 Epoch: 3, Average Loss: 3.605547330225375, Time taken: 292.08597803115845 seconds Epoch: 4, Loss: 3.522873640060425 Epoch: 4, Average Loss: 3.5965943490597687, Time taken: 292.3419635295868 seconds Epoch: 5, Loss: 3.5057291984558105 Epoch: 5, Average Loss: 3.5902317744244185, Time taken: 291.74957275390625 seconds Epoch: 6, Loss: 3.5161614418029785 Epoch: 6, Average Loss: 3.587415994346822, Time taken: 292.1076292991638 seconds Epoch: 7, Loss: 3.511284589767456 Epoch: 7, Average Loss: 3.5886631560416276, Time taken: 291.77211713790894 seconds Epoch: 8, Loss: 3.5088226795196533 Epoch: 8, Average Loss: 3.5876666606605734, Time taken: 291.85539841651917 seconds Epoch: 9, Loss: 3.505660057067871 Epoch: 9, Average Loss: 3.586097168831771, Time taken: 291.7574985027313 seconds Epoch: 10, Loss: 3.5023460388183594 Epoch: 10, Average Loss: 3.584595971687665, Time taken: 291.9380223751068 seconds Epoch: 11, Loss: 3.5048999786376953 Epoch: 11, Average Loss: 3.583113308641847, Time taken: 292.0109279155731 seconds Epoch: 12, Loss: 3.5031940937042236 Epoch: 12, Average Loss: 3.580842248840477, Time taken: 291.76497888565063 seconds Epoch: 13, Loss: 3.50352144241333 Epoch: 13, Average Loss: 3.5786439665370113, Time taken: 291.74229407310486 seconds Epoch: 14, Loss: 3.497807025909424 Epoch: 14, Average Loss: 3.5781566240035083, Time taken: 291.3640627861023 seconds Epoch: 15, Loss: 3.4952096939086914 Epoch: 15, Average Loss: 3.5777785469823917, Time taken: 291.2954170703888 seconds Epoch: 16, Loss: 3.497789144515991 Epoch: 16, Average Loss: 3.577572054282794, Time taken: 291.54339694976807 seconds Epoch: 17, Loss: 3.4942948818206787 Epoch: 17, Average Loss: 3.5770502394143167, Time taken: 291.37605381011963 seconds Epoch: 18, Loss: 3.4956154823303223 Epoch: 18, Average Loss: 3.576229152117392, Time taken: 290.98668670654297 seconds Epoch: 19, Loss: 3.492783546447754 Epoch: 19, Average Loss: 3.5748239583388934, Time taken: 290.95834708213806 seconds Epoch: 20, Loss: 3.4896273612976074 Epoch: 20, Average Loss: 3.572025065639626, Time taken: 290.86392188072205 seconds Epoch: 21, Loss: 3.4918222427368164 Epoch: 21, Average Loss: 3.5701150871501675, Time taken: 290.33885979652405 seconds Epoch: 22, Loss: 3.485771894454956 Epoch: 22, Average Loss: 3.568570154701349, Time taken: 289.31501388549805 seconds Epoch: 23, Loss: 3.4859931468963623 Epoch: 23, Average Loss: 3.567728873894695, Time taken: 289.6293840408325 seconds Epoch: 24, Loss: 3.4873950481414795 Epoch: 24, Average Loss: 3.567380349898973, Time taken: 289.5343222618103 seconds Epoch: 25, Loss: 3.4872074127197266 Epoch: 25, Average Loss: 3.5652468594308133, Time taken: 289.4293782711029 seconds Epoch: 26, Loss: 3.4843451976776123 Epoch: 26, Average Loss: 3.5643289098268225, Time taken: 289.92592668533325 seconds Epoch: 27, Loss: 3.4866209030151367 Epoch: 27, Average Loss: 3.563500219878135, Time taken: 289.3788197040558 seconds Epoch: 28, Loss: 3.481735944747925 Epoch: 28, Average Loss: 3.563423510740012, Time taken: 289.35951137542725 seconds Epoch: 29, Loss: 3.4829232692718506 Epoch: 29, Average Loss: 3.5658165968869575, Time taken: 289.45616269111633 seconds Epoch: 30, Loss: 3.487635374069214 Epoch: 30, Average Loss: 3.5698270244743435, Time taken: 288.84308218955994 seconds Epoch: 1, Loss: 3.4927570819854736 Epoch: 1, Average Train Loss: 3.5711315339962337, Average Valid Loss: 3.6290017239975207, Time taken: 312.00482964515686 seconds Epoch: 2, Loss: 3.489135503768921 Epoch: 2, Average Train Loss: 3.571447698788951, Average Valid Loss: 3.6297632943500173, Time taken: 312.0168857574463 seconds Epoch: 3, Loss: 3.4861600399017334 Epoch: 3, Average Train Loss: 3.5712698911079435, Average Valid Loss: 3.6287157372994856, Time taken: 311.7727360725403 seconds Epoch: 4, Loss: 3.481497049331665 Epoch: 4, Average Train Loss: 3.5710239242691504, Average Valid Loss: 3.625587866161809, Time taken: 311.834326505661 seconds Epoch: 5, Loss: 3.4813199043273926 Epoch: 5, Average Train Loss: 3.5704481733162594, Average Valid Loss: 3.6241414601152595, Time taken: 311.65282440185547 seconds Epoch: 6, Loss: 3.4821066856384277 Epoch: 6, Average Train Loss: 3.5700396979263074, Average Valid Loss: 3.624173494902524, Time taken: 311.28872323036194 seconds Epoch: 7, Loss: 3.4856691360473633 Epoch: 7, Average Train Loss: 3.569556606586442, Average Valid Loss: 3.6256951039487664, Time taken: 311.3145525455475 seconds Epoch: 8, Loss: 3.485272169113159 Epoch: 8, Average Train Loss: 3.5691312433648927, Average Valid Loss: 3.6258612672487893, Time taken: 311.2050988674164 seconds Epoch: 9, Loss: 3.483003616333008 Epoch: 9, Average Train Loss: 3.5687793647381745, Average Valid Loss: 3.62409432367845, Time taken: 311.68029260635376 seconds Epoch: 10, Loss: 3.4820141792297363 Epoch: 10, Average Train Loss: 3.5684012791049797, Average Valid Loss: 3.6230082620273936, Time taken: 311.22282123565674 seconds Epoch: 11, Loss: 3.4847073554992676 Epoch: 11, Average Train Loss: 3.567918264367281, Average Valid Loss: 3.624123398101691, Time taken: 310.99913930892944 seconds Epoch: 12, Loss: 3.4879515171051025 Epoch: 12, Average Train Loss: 3.567450798056425, Average Valid Loss: 3.625333491599921, Time taken: 310.98185873031616 seconds Epoch: 13, Loss: 3.485262393951416 Epoch: 13, Average Train Loss: 3.5671426356971945, Average Valid Loss: 3.6250820737896543, Time taken: 311.36055850982666 seconds Epoch: 14, Loss: 3.483027935028076 Epoch: 14, Average Train Loss: 3.565806134572047, Average Valid Loss: 3.622268743587263, Time taken: 311.6248605251312 seconds Epoch: 15, Loss: 3.4813599586486816 Epoch: 15, Average Train Loss: 3.5641346063903985, Average Valid Loss: 3.6236986445658133, Time taken: 311.5210154056549 seconds Epoch: 16, Loss: 3.4823060035705566 Epoch: 16, Average Train Loss: 3.5628339691307156, Average Valid Loss: 3.6223301923636235, Time taken: 311.5788004398346 seconds Epoch: 17, Loss: 3.482267141342163 Epoch: 17, Average Train Loss: 3.5627272088264785, Average Valid Loss: 3.6255278822147483, Time taken: 311.63774728775024 seconds Epoch: 18, Loss: 3.484595775604248 Epoch: 18, Average Train Loss: 3.5622248064882402, Average Valid Loss: 3.624577581882477, Time taken: 311.4593484401703 seconds Epoch: 19, Loss: 3.4822275638580322 Epoch: 19, Average Train Loss: 3.5611390410267356, Average Valid Loss: 3.624938767967802, Time taken: 311.02139925956726 seconds Epoch: 20, Loss: 3.4826202392578125 Epoch: 20, Average Train Loss: 3.5610133885430746, Average Valid Loss: 3.6282201156471716, Time taken: 311.3768696784973 seconds Epoch: 21, Loss: 3.479175090789795 Epoch: 21, Average Train Loss: 3.561582879875096, Average Valid Loss: 3.6201832077719946, Time taken: 310.644433259964 seconds Epoch: 22, Loss: 3.4808602333068848 Epoch: 22, Average Train Loss: 3.564162126965396, Average Valid Loss: 3.6246202606143374, Time taken: 310.5687975883484 seconds Epoch: 23, Loss: 3.4795570373535156 Epoch: 23, Average Train Loss: 3.5652912070995955, Average Valid Loss: 3.6242916367270728, Time taken: 310.62603640556335 seconds Epoch: 24, Loss: 3.481790065765381 Epoch: 24, Average Train Loss: 3.5656078845375845, Average Valid Loss: 3.623452957832452, Time taken: 311.1524329185486 seconds Epoch: 25, Loss: 3.4807748794555664 Epoch: 25, Average Train Loss: 3.565567792595113, Average Valid Loss: 3.6228528998114844, Time taken: 310.44076776504517 seconds Epoch: 26, Loss: 3.4812822341918945 Epoch: 26, Average Train Loss: 3.565412968737091, Average Valid Loss: 3.622271534168359, Time taken: 310.4774465560913 seconds Epoch: 27, Loss: 3.4829905033111572 Epoch: 27, Average Train Loss: 3.5651702132968395, Average Valid Loss: 3.621801087350556, Time taken: 310.73558282852173 seconds Epoch: 28, Loss: 3.4825916290283203 Epoch: 28, Average Train Loss: 3.5650173851745666, Average Valid Loss: 3.621133531584884, Time taken: 310.61714482307434 seconds Epoch: 29, Loss: 3.4815187454223633 Epoch: 29, Average Train Loss: 3.5648914144972883, Average Valid Loss: 3.620044368686098, Time taken: 310.6605079174042 seconds Epoch: 30, Loss: 3.4797537326812744 Epoch: 30, Average Train Loss: 3.564734306172273, Average Valid Loss: 3.6189178192254268, Time taken: 310.59446597099304 seconds Predicted output: <pad> <pad>