Add model

QuijoBERT/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "./QuijoBERT/backup",
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.19.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50000
+}

QuijoBERT/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65f7722ec4294cf9c4a092995573924d30d0a0a268b7e4db0c41a4ff2564b1c7
+size 327904939

QuijoBERT/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49120b72a0cb08b7a726d09864a5baba0e888193c56ff53895d356cc6cc501a
+size 3119

app.py

@@ -1,7 +1,43 @@
+# import gradio as gr
+
+# def greet(name):
+#     return "Hello Mr." + name + "!!"
+
+# iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+# iface.launch()
+
+
 import gradio as gr
+from numpy import kaiser
+
+from transformers import pipeline
+
+fill_mask = pipeline("fill-mask", model="./QuijoBERT", tokenizer = './QuijoBERT')
+
+def predict(text):
+    
+    res_dict = {}
+    x = fill_mask(text)
+    print('x')
+    for i in range(len(x)):
+        k = x[i]['sequence']
+        e = x[i]['score']
+        print(k, e) 
+        if e >= 0.05:
+            res_dict[k] = e
+    print (res_dict)
+    return res_dict
+    #return {x[0]["sequence"], x[0]["score"]}
+
+# texto = 'en un lugar de la <mask>'
+# print(predict(texto))
+
+iface = gr.Interface(
+    fn=predict, 
+    inputs='text',
+    outputs ='label',
+    examples=['En un lugar de la <mask>', 'En verdad, <mask> Sancho', 'Cómo has estado, bien mío, <mask> de mis ojos, compañero mío']
+)
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
 iface.launch()

quijoBERT.py

@@ -0,0 +1,113 @@
+
+
+from transformers import AutoTokenizer, AutoModelForMaskedLM,  RobertaConfig , RobertaTokenizer,RobertaForMaskedLM, DataCollatorForLanguageModeling, LineByLineTextDataset, Trainer, TrainingArguments
+
+
+from pathlib import Path
+from tokenizers import ByteLevelBPETokenizer
+from tokenizers.implementations import ByteLevelBPETokenizer
+from tokenizers.processors import BertProcessing
+import torch
+from torchinfo import summary
+
+
+import os
+
+paths = [str(x) for x in Path(".").glob("**/el_*.txt")]
+print(paths)
+# Initialize a tokenizer
+tokenizer = ByteLevelBPETokenizer()
+# Customize training
+tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2,
+special_tokens=[
+"<s>",
+"<pad>",
+"</s>",
+"<unk>",
+"<mask>",
+])
+
+
+dir_path = os.getcwd()
+token_dir = os.path.join(dir_path, 'QuijoBERT')
+
+if not os.path.exists(token_dir):
+  os.makedirs(token_dir)
+tokenizer.save_model('QuijoBERT')
+
+tokenizer = ByteLevelBPETokenizer(
+"./QuijoBERT/vocab.json",
+"./QuijoBERT/merges.txt",
+)
+
+tokenizer._tokenizer.post_processor = BertProcessing(
+("</s>", tokenizer.token_to_id("</s>")),
+("<s>", tokenizer.token_to_id("<s>")),
+)
+tokenizer.enable_truncation(max_length=512)
+
+
+
+config = RobertaConfig(
+  vocab_size=52_000,
+  max_position_embeddings=514,
+  num_attention_heads=12,
+  num_hidden_layers=6,
+  type_vocab_size=1,
+  )
+
+"""# Step 8: Re-creating the Tokenizer in Transformers"""
+
+tokenizer = RobertaTokenizer.from_pretrained("./QuijoBERT", max_length=512)
+
+#Initializing a Model 
+
+model = RobertaForMaskedLM(config=config)
+#In case we want to recover the after a crash
+#model = RobertaForMaskedLM.from_pretrained("./QuijoBERT/Checkpoint-xxxxx")
+
+
+#Tensorflow
+print(model)
+#Pytorch
+summary(model)
+
+
+dataset = LineByLineTextDataset(
+  tokenizer=tokenizer,
+  file_path="./el_quijote.txt",
+  block_size=128,
+  )
+
+
+#Defining a Data Collator
+
+data_collator = DataCollatorForLanguageModeling(
+  tokenizer=tokenizer, mlm=True, mlm_probability=0.15
+)
+
+# Initializing the Trainer Object
+training_args = TrainingArguments(
+  output_dir="./QuijoBERT",
+  overwrite_output_dir=True,
+  num_train_epochs=1,
+  per_device_train_batch_size=64,
+  save_steps=1000,
+  save_total_limit=2,
+  )
+trainer = Trainer(
+  model=model,
+  args=training_args,
+  data_collator=data_collator,
+  train_dataset=dataset,
+)
+
+
+#Training the Model
+print('aqui')
+trainer.train()
+trainer.save_model("./QuijoBERT")
+
+#Saving the Final Model(+tokenizer + config) to disk
+trainer.save_model("./QuijoBERT")
+