feat: Metrics ok

apis.py

@@ -45,6 +45,14 @@ class Response(BaseModel):
     """
     path: str = Query("", description="Style and sentiments of text")
     model : str = Query("whisper", description="Style and sentiments of text")
+class Response4(BaseModel):
+    path: str = Query("", description="path file")
+    system: str = Query("", description="prompt system LLM model with ocr and image claude")
+    content: str = Query("%s", description="prompt content LLM  model with ocr")
+    max_tokens: int = Query(1024, description="maxtoken LLM OCR model")
+    model: str = Query("Claude-sonnet", description="model")
+
+
 class Response1(BaseModel):
     path: str = Query("", description="path file")
     task_prompt: str = Query("", description="task of model")
@@ -66,7 +74,14 @@ class Response3(BaseModel):
     """
     path: str = Query("", description="Style and sentiments of text")
     Trusted: str = Query("", description="Style and sentiments of text")
-    mode : str = Query("whisper", description="Style and sentiments of text")
+    mode : str = Query("", description="Style and sentiments of text")
+
+class Response5(BaseModel):
+    """Structure of data to querry of make post from X or article blog
+    """
+    prompt: str = Query("", description="Style and sentiments of text")
+    mode : str = Query("", description="Style and sentiments of text")
+
 
 #Funcionales
 @app.get("/addTrusted")
@@ -99,11 +114,11 @@ def addTrusted(response:Response3):
             content={"content": "file no found" }
             )
     if mode_list[mode]=="texto":
-        hash1 = hashlib.sha256(path.encode()).hexdigest()+".txt"
+        info=str({"path":path,"trusted":Trusted,"mode":mode})
-        f = open("example/texto/"+hash1, "w")
+        hash1 = hashlib.sha256(info.encode()).hexdigest()
-        f.write(path)
+        # with open("example/texto/"+hash1, 'w') as f:
-        f.close()
+        #     json.dump(info, f)
-        path=pwd+"/"+pathText+hash1
+        # path=pwd+"/"+pathText+hash1
         length=len(Trusted)
         size=0
         duration=0
@@ -112,22 +127,63 @@ def addTrusted(response:Response3):
         size=file_stats.st_size / (1024 * 1024)
         length=0
         duration=0
+        hash1=""
     elif mode_list[mode]=="audio":
         with audioread.audio_open(path) as f: 
             duration = f.duration
             length=0
             size=0
-
+            hash1=""
     if db((db.trusted.path == path)&(db.trusted.mode == mode)).count()==0:
-        db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration=duration,last_modified=last_modified,length=length )
+        db.trusted.insert(path=path,trusted=Trusted,mode=mode,size=size,duration=duration,last_modified=last_modified,length=length,hash=hash1 )
         db.commit()
         return "Add %s in mode %s"%(path,mode)
     else:
         item=db((db.trusted.path == path)&(db.trusted.mode == mode)).select().last()
         modification_count=item.modification_count + 1
-        db((db.trusted.path == path)&(db.trusted.mode == mode)).update(trusted=Trusted,size=size,duration =duration,length=length,last_modified=last_modified,modification_count= modification_count)
+        db((db.trusted.path == path)&(db.trusted.mode == mode)).update(trusted=Trusted,size=size,duration =duration,length=length,last_modified=last_modified,modification_count= modification_count,hash=hash1)
         db.commit()
         return "Update %s in mode %s"%(path,mode)
+    
+
+@app.get("/addPrompt")
+@app.post("/addPrompt")
+def addPrompt(response:Response5):
+    """Api to add information of Trusted data
+
+    Args:
+        response (Response3): 3 params:
+        path : path of archive on system if is a file OR text if is text.
+        Trusted : information Trusted or better information in a process.
+        mode: llm_compra,llm_factura,llm_generaciontexto,llm_rag,ocr,voice,
+
+    Returns:
+        _type_: _description_
+    """
+    prompt=response.prompt
+    mode=response.mode
+    last_modified=datetime.now()
+    if mode not in mode_list.keys():
+        return JSONResponse(
+        status_code=404,
+        content={"content": "mode no found" }
+    )
+    if mode == "llm_compra" or mode == "llm_generaciontexto":
+        hash1 = str(hashlib.sha256(prompt.encode()).hexdigest())
+        # with open("example/texto/"+hash1, 'w') as f:
+        #     json.dump(info, f)
+        # path=pwd+"/"+pathText+hash1
+        length=len(prompt)
+    if db((db.prompt.hash == hash1)&(db.prompt.mode == mode)).count()==0:
+        db.prompt.insert(prompt=prompt,mode=mode,last_modified=last_modified,length=length,hash=hash1 )
+        db.commit()
+        return "Add %s in mode %s"%(prompt,mode)
+    else:
+        A=db((db.prompt.hash == hash1)&(db.prompt.mode == mode)).update(prompt=prompt,mode=mode,last_modified=last_modified,length=length+1,hash=hash1)
+        db.commit()
+        print(A,last_modified)
+        return "Update %s in mode %s"%(prompt,mode)
+
 
 @app.get("/EvalVoice")
 @app.post("/EvalVoice")
@@ -141,17 +197,18 @@ def EvalVoice(response:Response):
     )
     
     Trusted=db((db.trusted.path == path ) & ( db.trusted.mode == "voice")).select().last().trusted
-    print(Trusted)
     if model=="whisper":
         Sal=main.EvalWhisper(path,Trusted)
     else:
         Sal=main.EvalVosk(path,Trusted)
     Sal["last_modified"]=datetime.now()
-    if db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).count()==0:
+    if db((db.analitic_voice.path == Sal["path"]) & (db.analitic_voice.model == Sal["model"])).count()==0:
+        print(1,Sal)
         db.analitic_voice.insert(**Sal)
         db.commit()
     else:
-        db(db.analitic_voice.path == Sal["path"] and db.analitic_voice.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],last_modified=Sal["last_modified"])
+        print(2,Sal)
+        db((db.analitic_voice.path == Sal["path"]) & (db.analitic_voice.model == Sal["model"])).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],last_modified=Sal["last_modified"])
         db.commit()
     return Sal
 
@@ -192,7 +249,7 @@ def EvalVoicehtml():
     </style>
 </head>
 <body>
-    <h1>Petición POST a API</h1>
+    <h1>Petición Evaluar modelo de voz contra datos curados</h1>
 
     <select id="texto1">
         %s
@@ -237,6 +294,495 @@ def EvalVoicehtml():
     """%(Sal)
     return HTMLResponse(content=html, status_code=200)
 
+
+@app.get("/EvalLLMCompra")
+@app.post("/EvalLLMCompra")
+def EvalLLMCompra(response:Response4):
+    content=response.path
+    model=response.model
+    system= response.system
+    max_tokens= response.max_tokens
+    path=content
+    
+    if db((db.trusted.path == path ) & ( db.trusted.mode == "llm_compra")).count()==0:
+        return JSONResponse(
+        status_code=404,
+        content={"content": "Trusted no found" }
+    )
+    
+    Trusted=db((db.trusted.path == path ) & ( db.trusted.mode == "llm_compra")).select().last().trusted
+    Sal=main.EvalModelLLMCompra(system,content,model,max_tokens,Trusted)
+    Sal["last_modified"]=datetime.now()
+    if db((db.analitic_llm_compra.path == Sal["path"]) & (db.analitic_llm_compra.model == Sal["model"])).count()==0:
+        print(1,Sal)
+        db.analitic_llm_compra.insert(**Sal)
+        db.commit()
+    else:
+        print(2,Sal)
+        db((db.analitic_llm_compra.path == Sal["path"]) & (db.analitic_llm_compra.model == Sal["model"])).update(last_modified=Sal["last_modified"],relevance=Sal["relevance"],bias=Sal["bias"],toxic=Sal["toxic"],correctness=Sal["correctness"],relevance_r=Sal["relevance_r"],bias_r=Sal["bias_r"],toxic_r=Sal["toxic_r"],correctness_r=Sal["correctness_r"])
+        db.commit()
+    return Sal
+
+@app.get("/evalllmcomprahtml")
+def EvalLLMComprahtml():
+    dir_list = db((db.trusted.mode == "llm_compra" )).select()
+    Sal=""
+    t=1
+    for i in dir_list:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(i.path,str(t),str(i.path))
+        Sal=Sal+temp
+        t=t+1
+
+    dir_list2 = db((db.prompt.mode == "llm_compra" )).select()
+    Sal2=""
+    t=1
+    for i in dir_list2:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(i.prompt,str(t),str(i.prompt))
+        Sal2=Sal2+temp
+        t=t+1
+
+
+    html="""<!DOCTYPE html>
+<html lang="es">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Evaluacion de modelos voice2txt</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+        }
+        input, button {
+            margin: 10px 0;
+            padding: 5px;
+        }
+        #respuesta {
+            margin-top: 20px;
+            padding: 10px;
+            border: 1px solid #ccc;
+            background-color: #f9f9f9;
+        }
+    </style>
+</head>
+<body>
+    <h1>Petición Evaluar modelo de LLM para evaluar compras contra datos curados</h1>
+
+    <select id="texto1">
+        %s
+    </select>
+    
+    <br>
+    <select id="texto2">
+        <option value="meta-llama/Meta-Llama-3.1-70B-Instruct">meta-llama/Meta-Llama-3.1-70B-Instruct</option>
+        <option value="meta-llama/Meta-Llama-3.1-8B-Instruct">meta-llama/Meta-Llama-3.1-8B-Instruct</option>
+        <option value="Mistral">Mistral</option>
+    </select>
+        <br>
+    <select id="texto3">
+        %s
+    </select>
+        <br>
+    <input type="text" id="texto4" placeholder="max_tokens">
+    <br>
+    <button onclick="enviarPeticion()">Enviar petición</button>
+    <div id="respuesta"></div>
+
+    <script>
+        function enviarPeticion() {
+            const texto1 = document.getElementById('texto1').value;
+            const texto2 = document.getElementById('texto2').value;
+            const texto3 = document.getElementById('texto3').value;
+            const datos = {
+                path: texto1,
+                model: texto2,
+                system: texto3
+                
+
+            };
+
+            fetch('/EvalLLMCompra', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(datos)
+            })
+            .then(response => response.json())
+            .then(data => {
+                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
+            })
+            .catch(error => {
+                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
+            });
+        }
+    </script>
+</body>
+</html>
+    """%(Sal,Sal2)
+    return HTMLResponse(content=html, status_code=200)
+
+#
+@app.get("/EvalLLMGeneracionTexto")
+@app.post("/EvalLLMGeneracionTexto")
+def EvalLLMGeneracionTexto(response:Response4):
+    content=response.path
+    model=response.model
+    system= response.system
+    max_tokens= response.max_tokens
+    path=content
+    
+    if db((db.trusted.path == path ) & ( db.trusted.mode == "llm_generaciontexto")).count()==0:
+        return JSONResponse(
+        status_code=404,
+        content={"content": "Trusted no found" }
+    )
+    
+    Trusted=db((db.trusted.path == path ) & ( db.trusted.mode == "llm_generaciontexto")).select().last().trusted
+    Sal=main.EvalModelLLMCompra(system,content,model,max_tokens,Trusted)
+    Sal["last_modified"]=datetime.now()
+    if db((db.analitic_llm_generaciontexto.path == Sal["path"]) & (db.analitic_llm_generaciontexto.model == Sal["model"])).count()==0:
+        print(1,Sal)
+        db.analitic_llm_generaciontexto.insert(**Sal)
+        db.commit()
+    else:
+        print(2,Sal)
+        db((db.analitic_llm_generaciontexto.path == Sal["path"]) & (db.analitic_llm_generaciontexto.model == Sal["model"])).update(last_modified=Sal["last_modified"],relevance=Sal["relevance"],bias=Sal["bias"],toxic=Sal["toxic"],correctness=Sal["correctness"],relevance_r=Sal["relevance_r"],bias_r=Sal["bias_r"],toxic_r=Sal["toxic_r"],correctness_r=Sal["correctness_r"])
+        db.commit()
+    return Sal
+
+@app.get("/evalllmgeneraciontextohtml")
+def EvalLLMGeneracionTextohtml():
+    dir_list = db((db.trusted.mode == "llm_generaciontexto" )).select()
+    Sal=""
+    t=1
+    for i in dir_list:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(i.path,str(t),str(i.path))
+        Sal=Sal+temp
+        t=t+1
+
+    dir_list2 = db((db.prompt.mode == "llm_generaciontexto" )).select()
+    Sal2=""
+    t=1
+    for i in dir_list2:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(i.prompt,str(t),str(i.prompt))
+        Sal2=Sal2+temp
+        t=t+1
+
+
+    html="""<!DOCTYPE html>
+<html lang="es">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Evaluacion de modelos voice2txt</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+        }
+        input, button {
+            margin: 10px 0;
+            padding: 5px;
+        }
+        #respuesta {
+            margin-top: 20px;
+            padding: 10px;
+            border: 1px solid #ccc;
+            background-color: #f9f9f9;
+        }
+    </style>
+</head>
+<body>
+    <h1>Petición Evaluar modelo de LLM para generar texto contra datos curados</h1>
+
+    <select id="texto1">
+        %s
+    </select>
+    
+    <br>
+    <select id="texto2">
+        <option value="meta-llama/Meta-Llama-3.1-70B-Instruct">meta-llama/Meta-Llama-3.1-70B-Instruct</option>
+        <option value="meta-llama/Meta-Llama-3.1-8B-Instruct">meta-llama/Meta-Llama-3.1-8B-Instruct</option>
+        <option value="Mistral">Mistral</option>
+    </select>
+        <br>
+    <select id="texto3">
+        %s
+    </select>
+        <br>
+    <input type="text" id="texto4" placeholder="max_tokens">
+    <br>
+    <button onclick="enviarPeticion()">Enviar petición</button>
+    <div id="respuesta"></div>
+
+    <script>
+        function enviarPeticion() {
+            const texto1 = document.getElementById('texto1').value;
+            const texto2 = document.getElementById('texto2').value;
+            const texto3 = document.getElementById('texto3').value;
+            const datos = {
+                path: texto1,
+                model: texto2,
+                system: texto3
+                
+
+            };
+
+            fetch('/EvalLLMGeneracionTexto', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(datos)
+            })
+            .then(response => response.json())
+            .then(data => {
+                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
+            })
+            .catch(error => {
+                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
+            });
+        }
+    </script>
+</body>
+</html>
+    """%(Sal,Sal2)
+    return HTMLResponse(content=html, status_code=200)
+
+
+
+#
+
+@app.get("/EvalFact")
+@app.post("/EvalFact")
+def EvalFact(response:Response1):
+    path=response.path
+    task_prompt=response.task_prompt
+    option=response.model
+    TrustedOCR=response.TrustedOCR
+    Trusted=TrustedOCR
+    if task_prompt=="":
+        if Trusted=="":
+            row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first()
+            try:
+                Trusted=row.trusted
+            except:
+                pass
+    Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option)
+    Sal["path"]=path
+    if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0:
+        db.analitic_ocr.insert(**Sal)
+        db.commit()
+    else:
+        db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"])
+        db.commit()
+    return Sal
+
+
+@app.get("/evalocrfactura")
+def EvalOCRFactura():
+    dir_list = os.listdir(pathFact)
+    Sal=""
+    t=1
+    for i in dir_list:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
+        Sal=Sal+temp
+        t=t+1
+    html="""<!DOCTYPE html>
+<html lang="es">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Evaluacion de modelos OCR</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+        }
+        input, button {
+            margin: 10px 0;
+            padding: 5px;
+        }
+        #respuesta {
+            margin-top: 20px;
+            padding: 10px;
+            border: 1px solid #ccc;
+            background-color: #f9f9f9;
+        }
+    </style>
+</head>
+<body>
+    <h1>Petición POST a API</h1>
+    <select id="texto1">
+        %s
+    </select>
+    <br>
+
+    <select id="texto2">
+        <option value="More Detailed Caption">More Detailed Caption</option>
+        <option value="OCR">OCR</option>
+        <option value="parsed">parsed</option>
+        <option value="scan">scan</option>
+    </select>
+    <br>
+    <input type="text" id="texto3" placeholder="TrustedOCR">
+    <br>
+    <input type="text" id="texto4" placeholder="option">
+    <br>
+    <button onclick="enviarPeticion()">Enviar petición</button>
+    <div id="respuesta"></div>
+
+    <script>
+        function enviarPeticion() {
+            const texto1 = document.getElementById('texto1').value;
+            const texto2 = document.getElementById('texto2').value;
+            const texto3 = document.getElementById('texto3').value;
+            const texto4 = document.getElementById('texto4').value;
+            const datos = {
+                path: texto1,
+                task_prompt: texto2,
+                TrustedOCR: texto3,
+                option: texto4
+            };
+
+            fetch('/EvalFact', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(datos)
+            })
+            .then(response => response.json())
+            .then(data => {
+                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
+            })
+            .catch(error => {
+                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
+            });
+        }
+    </script>
+</body>
+</html>
+    """%(Sal)
+    return HTMLResponse(content=html, status_code=200)
+
+
+
+@app.get("/evalllmfacturas")
+def EvalllmFacturas():
+    dir_list = os.listdir(pathFact)
+    Sal=""
+    t=1
+    for i in dir_list:
+        temp="""<option value="%s">Opción %s, %s</option>
+        """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
+        Sal=Sal+temp
+        t=t+1
+    html="""<!DOCTYPE html>
+<html lang="es">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Evaluacion modelos LLM</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+        }
+        input, button {
+            margin: 10px 0;
+            padding: 5px;
+        }
+        #respuesta {
+            margin-top: 20px;
+            padding: 10px;
+            border: 1px solid #ccc;
+            background-color: #f9f9f9;
+        }
+    </style>
+</head>
+<body>
+    <h1>Petición POST a API</h1>
+    <select id="texto1">
+        %s
+    </select>
+    <br>
+
+    <select id="texto2">
+        <option value="">N.A.</option>
+        <option value="More Detailed Caption">More Detailed Caption</option>
+        <option value="OCR">OCR</option>
+        <option value="parsed">parsed</option>
+        <option value="scan">scan</option>
+    </select>
+    <br>
+    <input type="text" id="texto3" placeholder="system" value="Eres un chatbot amable">
+    <br>
+    <input type="text" id="texto4" placeholder="content" value="%s">
+    <br>
+    <input type="number" id="texto5" placeholder="max_tokens" value=1024>
+    <br>
+        <input type="text" id="texto6" placeholder="model" value="Claude-sonnet">
+    <br>
+        <input type="text" id="texto7" placeholder="prompt" value="Analiza la factura">
+    <br>
+    <input type="text" id="texto8" placeholder="TrustedLLmjson" value="{'A':''}">
+    <br>
+    <button onclick="enviarPeticion()">Enviar petición</button>
+    <div id="respuesta"></div>
+
+    <script>
+        function enviarPeticion() {
+            const texto1 = document.getElementById('texto1').value;
+            const texto2 = document.getElementById('texto2').value;
+            const texto3 = document.getElementById('texto3').value;
+            const texto4 = document.getElementById('texto4').value;
+            const texto5 = document.getElementById('texto5').value;
+            const texto6 = document.getElementById('texto6').value;
+            const texto7 = document.getElementById('texto7').value;
+            const texto8 = document.getElementById('texto8').value;
+ 
+            const datos = {
+                path: texto1,
+                task_prompt: texto2,
+                system: texto3,
+                content:texto4,
+                max_tokens:texto5,
+                model:texto6,
+                prompt:texto7,
+                TrustedLLmjson:texto8,
+            };
+
+            fetch('/EvalLLMFact', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json'
+                },
+                body: JSON.stringify(datos)
+            })
+            .then(response => response.json())
+            .then(data => {
+                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
+            })
+            .catch(error => {
+                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
+            });
+        }
+    </script>
+</body>
+</html>
+    """%(Sal,"%s")
+    return HTMLResponse(content=html, status_code=200)
+
+
+
+
 #Por revisar
 
 def list2tablehtml(listdata,model):
@@ -385,33 +931,8 @@ display:flex;
 
 
 
-@app.get("/EvalFact")
-@app.post("/EvalFact")
-def EvalFact(response:Response1):
-    path=response.path
-    task_prompt=response.task_prompt
-    option=response.model
-    TrustedOCR=response.TrustedOCR
-    Trusted=TrustedOCR
-    if task_prompt=="":
-        if Trusted=="":
-            row=db(db.trusted.path == path and db.trusted.mode == "OCR").select().first()
-            try:
-                Trusted=row.trusted
-            except:
-                pass
-    Sal=main.EvalFacturas(path,task_prompt,TrustedOCR,option)
-    Sal["path"]=path
-    if db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).count()==0:
-        db.analitic_ocr.insert(**Sal)
-        db.commit()
-    else:
-        db(db.analitic_ocr.path == Sal["path"] and db.analitic_ocr.model == Sal["model"]).update(similarity= Sal["similarity"],similaritypartial= Sal["similaritypartial"],jsonok=Sal["jsonok"])
-        db.commit()
 
 
-    return Sal
-
 @app.get("/EvalLLMFact")
 @app.post("/EvalLLMFact")
 def EvalLLMFact(response:Response2):
@@ -429,93 +950,7 @@ def EvalLLMFact(response:Response2):
 
 
 
-@app.get("/evalocrfactura")
-def EvalOCRFactura():
-    dir_list = os.listdir(pathFact)
-    Sal=""
-    t=1
-    for i in dir_list:
-        temp="""<option value="%s">Opción %s, %s</option>
-        """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
-        Sal=Sal+temp
-        t=t+1
-    html="""<!DOCTYPE html>
-<html lang="es">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Evaluacion de modelos OCR</title>
-    <style>
-        body {
-            font-family: Arial, sans-serif;
-            margin: 20px;
-        }
-        input, button {
-            margin: 10px 0;
-            padding: 5px;
-        }
-        #respuesta {
-            margin-top: 20px;
-            padding: 10px;
-            border: 1px solid #ccc;
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Petición POST a API</h1>
-    <select id="texto1">
-        %s
-    </select>
-    <br>
 
-    <select id="texto2">
-        <option value="More Detailed Caption">More Detailed Caption</option>
-        <option value="OCR">OCR</option>
-        <option value="parsed">parsed</option>
-        <option value="scan">scan</option>
-    </select>
-    <br>
-    <input type="text" id="texto3" placeholder="TrustedOCR">
-    <br>
-    <input type="text" id="texto4" placeholder="option">
-    <br>
-    <button onclick="enviarPeticion()">Enviar petición</button>
-    <div id="respuesta"></div>
-
-    <script>
-        function enviarPeticion() {
-            const texto1 = document.getElementById('texto1').value;
-            const texto2 = document.getElementById('texto2').value;
-            const texto3 = document.getElementById('texto3').value;
-            const texto4 = document.getElementById('texto4').value;
-            const datos = {
-                path: texto1,
-                task_prompt: texto2,
-                TrustedOCR: texto3,
-                option: texto4
-            };
-
-            fetch('/EvalFact', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                body: JSON.stringify(datos)
-            })
-            .then(response => response.json())
-            .then(data => {
-                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
-            })
-            .catch(error => {
-                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
-            });
-        }
-    </script>
-</body>
-</html>
-    """%(Sal)
-    return HTMLResponse(content=html, status_code=200)
 
 def list2tablehtmlOCR(listdata,model):
     html="""<h2>Table of {0}</h2>
@@ -651,108 +1086,3 @@ display:flex;
 
 
 
-@app.get("/evalllmfacturas")
-def EvalllmFacturas():
-    dir_list = os.listdir(pathFact)
-    Sal=""
-    t=1
-    for i in dir_list:
-        temp="""<option value="%s">Opción %s, %s</option>
-        """%(str(pwd+"/"+pathFact+"/"+i),str(t),str(i))
-        Sal=Sal+temp
-        t=t+1
-    html="""<!DOCTYPE html>
-<html lang="es">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Evaluacion modelos LLM</title>
-    <style>
-        body {
-            font-family: Arial, sans-serif;
-            margin: 20px;
-        }
-        input, button {
-            margin: 10px 0;
-            padding: 5px;
-        }
-        #respuesta {
-            margin-top: 20px;
-            padding: 10px;
-            border: 1px solid #ccc;
-            background-color: #f9f9f9;
-        }
-    </style>
-</head>
-<body>
-    <h1>Petición POST a API</h1>
-    <select id="texto1">
-        %s
-    </select>
-    <br>
-
-    <select id="texto2">
-        <option value="">N.A.</option>
-        <option value="More Detailed Caption">More Detailed Caption</option>
-        <option value="OCR">OCR</option>
-        <option value="parsed">parsed</option>
-        <option value="scan">scan</option>
-    </select>
-    <br>
-    <input type="text" id="texto3" placeholder="system" value="Eres un chatbot amable">
-    <br>
-    <input type="text" id="texto4" placeholder="content" value="%s">
-    <br>
-    <input type="number" id="texto5" placeholder="max_tokens" value=1024>
-    <br>
-        <input type="text" id="texto6" placeholder="model" value="Claude-sonnet">
-    <br>
-        <input type="text" id="texto7" placeholder="prompt" value="Analiza la factura">
-    <br>
-    <input type="text" id="texto8" placeholder="TrustedLLmjson" value="{'A':''}">
-    <br>
-    <button onclick="enviarPeticion()">Enviar petición</button>
-    <div id="respuesta"></div>
-
-    <script>
-        function enviarPeticion() {
-            const texto1 = document.getElementById('texto1').value;
-            const texto2 = document.getElementById('texto2').value;
-            const texto3 = document.getElementById('texto3').value;
-            const texto4 = document.getElementById('texto4').value;
-            const texto5 = document.getElementById('texto5').value;
-            const texto6 = document.getElementById('texto6').value;
-            const texto7 = document.getElementById('texto7').value;
-            const texto8 = document.getElementById('texto8').value;
- 
-            const datos = {
-                path: texto1,
-                task_prompt: texto2,
-                system: texto3,
-                content:texto4,
-                max_tokens:texto5,
-                model:texto6,
-                prompt:texto7,
-                TrustedLLmjson:texto8,
-            };
-
-            fetch('/EvalLLMFact', {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                body: JSON.stringify(datos)
-            })
-            .then(response => response.json())
-            .then(data => {
-                document.getElementById('respuesta').innerHTML = JSON.stringify(data, null, 2);
-            })
-            .catch(error => {
-                document.getElementById('respuesta').innerHTML = 'Error: ' + error;
-            });
-        }
-    </script>
-</body>
-</html>
-    """%(Sal,"%s")
-    return HTMLResponse(content=html, status_code=200)

databases.py

@@ -9,8 +9,20 @@ db.define_table(
     Field("sizeMB",type="double",default=0),# audio,factura
     Field("length",type="integer",default=0),#texto
     Field('last_modified', 'datetime'),
-    Field('modification_count', 'integer', default=0)
+    Field('modification_count', 'integer', default=0),
+    Field('hash')
 )
+
+db.define_table(
+    "prompt",
+    Field("prompt"),
+    Field("mode"),
+    Field("length",type="integer",default=0),
+    Field('hash',unique=True),
+    Field('last_modified', 'datetime'),
+)
+
+
 db.define_table(
     "analitic_voice",
     Field("content"),
@@ -43,8 +55,14 @@ db.define_table(
     Field("model"),
     Field("time", type="double"),
     Field("path"),
-    Field("similarity", type="double"),
+    Field("relevance", type="double"),
-    Field("similaritypartial", type="double"),
+    Field("bias", type="double"),
+    Field("toxic", type="double"),
+    Field("correctness", type="double"),
+    Field("relevance_r"),
+    Field("bias_r"),
+    Field("toxic_r"),
+    Field("correctness_r"),
     Field('last_modified', 'datetime')    
 )
 
@@ -67,9 +85,15 @@ db.define_table(
     Field("model"),
     Field("time", type="double"),
     Field("path"),
-    Field("similarity", type="double"),
+    Field("relevance", type="double"),
-    Field("similaritypartial", type="double"),
+    Field("bias", type="double"),
-    Field('last_modified', 'datetime')    
+    Field("toxic", type="double"),
+    Field("correctness", type="double"),
+    Field("relevance_r"),
+    Field("bias_r"),
+    Field("toxic_r"),
+    Field("correctness_r"),
+    Field('last_modified', 'datetime')   
 )
 
 db.define_table(

gui.py

@@ -7,12 +7,9 @@ import pandas as pd
 import requests
 import statistics
 from databases import db
+import time
 pwd = os.getcwd()
 
-HTML = os.path.join(pwd,"html", "index.html")
-file_read = codecs.open(HTML, "r", "utf-8")
-index = file_read.read()
-html_page_index = Html(index)
 def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"):
     configPath=os.path.join(os.getcwd(),relPath)
     with open(configPath, 'r', encoding='utf-8') as file:
@@ -20,6 +17,8 @@ def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experime
     Output= config[dataOut]
     return Output
 mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list")
+
+
 def getmetricvoice(model):
     rows = db(db.analitic_voice.model==model).select()
     rows_list = rows.as_list()
@@ -36,70 +35,247 @@ def getmetricvoice(model):
 
 def html_getmetricvoice():
     models=list()
+    t=time.time()
     for row in db().select(db.analitic_voice.model, distinct=True):
         models.append(row.model)
     data={}
     for model in models:
         data[model]=getmetricvoice(model)
     data=pd.DataFrame(data).T
-    datafiles={}
+    data_files={}
     for row in db().select(db.analitic_voice.ALL):
-        datafiles[row.id]=row.as_dict()
+        data_files[row.id]=row.as_dict()
-    datafiles=pd.DataFrame(datafiles).T
+
+    data_files=pd.DataFrame(data_files).T
+
+    #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
+                       #columns=['model'], aggfunc="sum")
+ 
+
     html="""
+    <h1>Data general de los modelos</h1>
     <taipy:table>{data_voice}</taipy:table>
+    <h1>Data de cada muestra</h1>
     <taipy:table filter=True>{data_files_voice}</taipy:table>
-    """
-
-    return html,data,datafiles
-html_page_getmetricsvoice,data_voice,data_files_voices=html_getmetricvoice()
-
-mode="voice"
-modetypedata="audio"
-file="id2"
-def changemenu(mode):
-    if mode_list[mode]=="audio":
-        pathori="example/audio"
-    if mode_list[mode]=="factura":
-        pathori="example/factura"
-    if mode_list[mode]=="texto":
-        pathori="example/texto"
-    seltypedata=mode_list[mode]  
-    dir_list = os.listdir(pathori)
     
-    return pathori,seltypedata,dir_list
+
+    """
+    #<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
+
+    return html,data,data_files
 
 
-def trustedallhtml(mode):
-    pathori,seltypedata,dir_list=changemenu(mode)
 
 
-    textmode=""
-    for modeused in mode_list.keys():
-        textmode=textmode+"('%s','%s'),"%(modeused,modeused)
-    html="""<taipy:selector lov="{[%s]}" dropdown True on_change=changemenu>{sel}</taipy:selector>"""%(textmode) 
-    Sal=""
-    for i in dir_list:
-        temp="""('%s', '%s'),"""%(str(pwd+"/"+pathori+"/"+i),str(i))
-        Sal=Sal+temp
-    html2="""<taipy:selector lov="{[%s]}" dropdown True >{sel2}</taipy:selector>"""%(Sal)
-    return html+html2
 
-html_page_trustedall = Html(trustedallhtml(mode))
+def getmetricllm_compra(model):
-#print(sel,sel2,seltypedata)
+    rows = db(db.analitic_llm_compra.model==model).select()
-HTML = os.path.join(pwd,"html", "index.html")
+    rows_list = rows.as_list()
-file_read = codecs.open(HTML, "r", "utf-8")
+    data=pd.DataFrame(rows_list)
-index = file_read.read()
+
-html_page_index = Html(index)
+    #durationL=list()
+    #for i in rows_list:
+        #durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
+    #duration=statistics.mean(durationL)
+    time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0]
+    relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0]
+    bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0]
+    toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0]
+
+    correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0]
+    #similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
+    #similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
+    #efectivetime=time/duration
+    return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness})
+
+def html_getmetricllm_compra():
+    models=list()
+    t=time.time()
+    for row in db().select(db.analitic_llm_compra.model, distinct=True):
+        models.append(row.model)
+    data={}
+    for model in models:
+        data[model]=getmetricllm_compra(model)
+    data=pd.DataFrame(data).T
+    data_files={}
+    for row in db().select(db.analitic_llm_compra.ALL):
+        data_files[row.id]=row.as_dict()
+
+    data_files=pd.DataFrame(data_files).T
+
+    #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
+                       #columns=['model'], aggfunc="sum")
+
+
+    html="""
+    <h1>Data general de los modelos</h1>
+    <taipy:table>{data_llm_compra}</taipy:table>
+    <h1>Data de cada muestra</h1>
+    <taipy:table filter=True >{data_files_llm_compra}</taipy:table>
+    
+
+    """
+    #<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
+
+    return html,data,data_files
+
+
+def getmetricllm_generaciontexto(model):
+    rows = db(db.analitic_llm_generaciontexto.model==model).select()
+    rows_list = rows.as_list()
+    data=pd.DataFrame(rows_list)
+
+    #durationL=list()
+    #for i in rows_list:
+        #durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
+    #duration=statistics.mean(durationL)
+    time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0]
+    relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0]
+    bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0]
+    toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0]
+
+    correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0]
+    #similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
+    #similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
+    #efectivetime=time/duration
+    return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness})
+
+def html_getmetricllm_generaciontexto():
+    models=list()
+    t=time.time()
+    for row in db().select(db.analitic_llm_generaciontexto.model, distinct=True):
+        models.append(row.model)
+    data={}
+    for model in models:
+        data[model]=getmetricllm_generaciontexto(model)
+    data=pd.DataFrame(data).T
+    data_files={}
+    for row in db().select(db.analitic_llm_generaciontexto.ALL):
+        data_files[row.id]=row.as_dict()
+
+    data_files=pd.DataFrame(data_files).T
+
+    #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
+                       #columns=['model'], aggfunc="sum")
+
+
+    html="""
+    <h1>Data general de los modelos</h1>
+    <taipy:table>{data_llm_generaciontexto}</taipy:table>
+    <h1>Data de cada muestra</h1>
+    <taipy:table filter=True >{data_files_llm_generaciontexto}</taipy:table>
+    
+
+    """
+    #<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
+
+    return html,data,data_files
+
+
+
+
+def getmetricllm_factura(model):
+    rows = db(db.analitic_llm_factura.model==model).select()
+    rows_list = rows.as_list()
+    data=pd.DataFrame(rows_list)
+
+    #durationL=list()
+    #for i in rows_list:
+        #durationL.append(db(db.trusted.path == i["path"] ).select().last().duration)
+    #duration=statistics.mean(durationL)
+    time=pd.pivot_table(data,values=['time'],index="model")['time'].values[0]
+    relevance=pd.pivot_table(data,values=["relevance"],index="model")['relevance'].values[0]
+    bias=pd.pivot_table(data,values=["bias"],index="model")['bias'].values[0]
+    toxic=pd.pivot_table(data,values=["toxic"],index="model")['toxic'].values[0]
+
+    correctness=pd.pivot_table(data,values=["correctness"],index="model")['correctness'].values[0]
+    #similarity=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similarity'].values[0]
+    #similaritypartial=pd.pivot_table(data,values=['time','similarity', 'similaritypartial'],index="model")['similaritypartial'].values[0]
+    #efectivetime=time/duration
+    return ({"model":model,"time":time,"relevance":relevance,"bias":bias,"toxic":toxic,"correctness":correctness})
+
+
+
+def html_getmetricllm_factura():
+    models=list()
+    t=time.time()
+    for row in db().select(db.analitic_llm_factura.model, distinct=True):
+        models.append(row.model)
+    data={}
+    for model in models:
+        data[model]=getmetricllm_factura(model)
+    data=pd.DataFrame(data).T
+    data_files={}
+    for row in db().select(db.analitic_llm_factura.ALL):
+        data_files[row.id]=row.as_dict()
+
+    data_files=pd.DataFrame(data_files).T
+
+    #table = pd.pivot_table(data_files, values=['path', 'similarity','similaritypartial'], index=['path'],
+                       #columns=['model'], aggfunc="sum")
+
+
+    html="""
+    <h1>Data general de los modelos</h1>
+    <taipy:table>{data_llm_factura}</taipy:table>
+    <h1>Data de cada muestra</h1>
+    <taipy:table filter=True >{data_files_llm_factura}</taipy:table>
+    
+
+    """
+    #<taipy:chart mode="markers" x="x" y[1]="time" y[2]="similarity">{data_files_voice}</taipy:chart>
+
+    return html,data,data_files
+
+
+
+
+
+def on_init(state):
+    state.html_page_getmetricsvoice,state.data_voice,state.data_files_voice=html_getmetricvoice()
+    state.html_page_getmetricsllm_compra,state.data_llm_compra,state.data_files_llm_compra=html_getmetricllm_compra()
+    state.html_page_getmetricsllm_generaciontexto,state.data_llm_generaciontexto,state.data_files_llm_generaciontexto=html_getmetricllm_generaciontexto()
+    state.html_page_getmetricsllm_factura,state.data_llm_factura,state.data_files_llm_factura=html_getmetricllm_factura()
+
+    pass
+    
+    
+
+html_page_getmetricsvoice,data_voice,data_files_voice=html_getmetricvoice()
+
+html_page_getmetricsllm_compra,data_llm_compra,data_files_llm_compra=html_getmetricllm_compra()
+
+html_page_getmetricsllm_generaciontexto,data_llm_generaciontexto,data_files_llm_generaciontexto=html_getmetricllm_generaciontexto()
+
+html_page_getmetricsllm_factura,data_llm_factura,data_files_llm_factura=html_getmetricllm_factura()
+# mode="voice"
+# modetypedata="audio"
+# file="id2"
+# def changemenu(mode):
+#     if mode_list[mode]=="audio":
+#         pathori="example/audio"
+#     if mode_list[mode]=="factura":
+#         pathori="example/factura"
+#     if mode_list[mode]=="texto":
+#         pathori="example/texto"
+#     seltypedata=mode_list[mode]  
+#     dir_list = os.listdir(pathori)
+    
+#     return pathori,seltypedata,dir_list
+
+
+
+
 
-data=pd.DataFrame({'col1': [1, 2], 'col2': [3, 4]})
 
 pages = {
-  "/": html_page_index ,
   "getmetricsvoice": Html(html_page_getmetricsvoice),
-  "trustedall":html_page_trustedall
+  "getmetricsllm_compra": Html(html_page_getmetricsllm_compra),
+  "getmetricsllm_generaciontexto": Html(html_page_getmetricsllm_generaciontexto),
+    "getmetricsllm_factura": Html(html_page_getmetricsllm_factura)
 }
 
 app = Gui(pages=pages)
+app.on_init=on_init
 if __name__=="__main__":
-    app.run(use_reloader=True,port=7882, change_delay=1600)#state.imageActive2,
+    app.run(use_reloader=True,port=7882)#state.imageActive2,

main.py

@@ -2,24 +2,37 @@ import requests
 import evaluate
 import deepdiff
 import json
+import os 
+
 from fuzzywuzzy import fuzz
 from deepdiff import DeepDiff
 from deepdiff import Delta
 import databases
+import metrics
 #print(evaluate.list_evaluation_modules())
+pwd = os.getcwd()
 urlAud="http://127.0.0.1:7870/"
 urlText="http://127.0.0.1:7869"
-password="1223Aer*"
+
+
+def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"):
+    configPath=os.path.join(os.getcwd(),relPath)
+    with open(configPath, 'r', encoding='utf-8') as file:
+        config = json.load(file)[nameModel]
+    Output= config[dataOut]
+    return Output
+mode_list=extractConfig(nameModel="SystemData",dataOut="mode_list")
+keyanthropic=extractConfig(nameModel="SystemData",dataOut="keyantrophics")
+password=extractConfig(nameModel="SystemData",dataOut="password")
+
+
 def EvalVoice2Text(endpoint,datajson,Trusted):
     """Evaluate Voice 2 text
     """
     apiUrl=urlAud+endpoint
     response = requests.get(apiUrl,  json=datajson)
-    print(datajson)
     A=json.loads(response.content)
-    print(A)
     time=A['time']
-
     similarity=fuzz.ratio( Trusted.strip().lower(),A['message'].strip().lower())
     similarityPartial=fuzz.partial_ratio( Trusted.strip().lower(),A['message'].strip().lower())
     path=datajson["local"]
@@ -34,27 +47,105 @@ def EvalVoice2Text(endpoint,datajson,Trusted):
             "path":path
             }
 
-
 def EvalWhisper(path,Trusted=""):
     endpoint="/voice2txt"
     datajson={"url":"","password":password ,"model":"whisper","local":path}
     return EvalVoice2Text(endpoint,datajson,Trusted)
 
-
-# EvalWhisper(path="example/AwACAgEAAxkBAAIBw2YX8o2vGGCNtZCXk7mY1Bm5w__lAAJmBAACxe7ARI1fUWAGcz_RNAQ.ogg",
-#             Trusted="Hoy compre dos medicinas Tereleji en Cruz Verde por un monto de 494 mil 400 pesos colombianos.",
-#             endpoint="/voice2txt")
-
 def EvalVosk(path,Trusted=""):
     endpoint="/voice2txtlocal"
     datajson={"url":"","password":password ,"model":"models/vosk-model-small-es-0.42","local":path}
     return EvalVoice2Text(endpoint,datajson,Trusted)
 
 
+def EvalLLMCompra(endpoint,datajson,Trusted):
+    """Evaluate LLL compra
+    """
+    apiUrl=urlText+endpoint
+    response = requests.get(apiUrl,  json=datajson)
+    A=json.loads(response.content)
+    time=A['time']
+    relevance=metrics.RelevanceMetric(datajson["system"]+datajson["content"],response.content)
+    bias=metrics.BiasMetric22(datajson["system"]+datajson["content"],response.content)
+    toxic=metrics.ToxicMetric(datajson["system"]+datajson["content"],response.content)
+    correctness=metrics.correctnessMetric(datajson["system"]+datajson["content"],response.content,Trusted)
+    #jsonmetrics=metrics.jsonMetrics(response.content,Trusted)
+    #similarity=fuzz.ratio( Trusted.strip().lower(),A['content'].strip().lower())
+    #similarityPartial=fuzz.partial_ratio( Trusted.strip().lower(),A['content'].strip().lower())
+    #path=datajson["local"]
+    model=datajson["model"]
+
+    message=A['content']
+    return {"content":message,
+            "trusted":Trusted,
+            "model":model,
+            "time":time,
+            "relevance":relevance["score"],
+            "bias":bias["score"],
+            "toxic":toxic["score"],
+            "correctness":correctness["score"],
+            "relevance_r":relevance["reason"],
+            "bias_r":bias["reason"],
+            "toxic_r":toxic["reason"],
+            "correctness_r":correctness["reason"],
+            "path":message
+            }
+
+def EvalModelLLMCompra(system,content,model,max_new_tokens,Trusted):
+    endpoint="/genTextCustom"
+    datajson={"system":system,"content":content,"password":password ,"model":model,"max_new_token":max_new_tokens}
+    return EvalLLMCompra(endpoint,datajson,Trusted)
+
+def EvalLLMGeneracionTexto(endpoint,datajson,Trusted):
+    """Evaluate LLL compra
+    """
+    apiUrl=urlText+endpoint
+    response = requests.get(apiUrl,  json=datajson)
+    A=json.loads(response.content)
+    time=A['time']
+    relevance=metrics.RelevanceMetric(datajson["system"]+datajson["content"],response.content)
+    bias=metrics.BiasMetric22(datajson["system"]+datajson["content"],response.content)
+    toxic=metrics.ToxicMetric(datajson["system"]+datajson["content"],response.content)
+    correctness=metrics.correctnessMetric(datajson["system"]+datajson["content"],response.content,Trusted)
+    #jsonmetrics=metrics.jsonMetrics(response.content,Trusted)
+    #similarity=fuzz.ratio( Trusted.strip().lower(),A['content'].strip().lower())
+    #similarityPartial=fuzz.partial_ratio( Trusted.strip().lower(),A['content'].strip().lower())
+    #path=datajson["local"]
+    model=datajson["model"]
+
+    message=A['content']
+    return {"content":message,
+            "trusted":Trusted,
+            "model":model,
+            "time":time,
+            "relevance":relevance["score"],
+            "bias":bias["score"],
+            "toxic":toxic["score"],
+            "correctness":correctness["score"],
+            "relevance_r":relevance["reason"],
+            "bias_r":bias["reason"],
+            "toxic_r":toxic["reason"],
+            "correctness_r":correctness["reason"],
+            "path":message
+            }
+
+def EvalModelLLMGeneracionTexto(system,content,model,max_new_tokens,Trusted):
+    endpoint="/genTextCustom"
+    datajson={"system":system,"content":content,"password":password ,"model":model,"max_new_token":max_new_tokens}
+    return EvalLLMGeneracionTexto(endpoint,datajson,Trusted)
+
+
+
+
+
+
 
 # EvalVosk(path="example/AwACAgEAAxkBAAIBw2YX8o2vGGCNtZCXk7mY1Bm5w__lAAJmBAACxe7ARI1fUWAGcz_RNAQ.ogg",
 #             Trusted="Hoy compre dos medicinas Tereleji en Cruz Verde por un monto de 494 mil 400 pesos colombianos.",
 #             endpoint="/voice2txtlocal")
+# EvalWhisper(path="example/AwACAgEAAxkBAAIBw2YX8o2vGGCNtZCXk7mY1Bm5w__lAAJmBAACxe7ARI1fUWAGcz_RNAQ.ogg",
+#             Trusted="Hoy compre dos medicinas Tereleji en Cruz Verde por un monto de 494 mil 400 pesos colombianos.",
+#             endpoint="/voice2txt")
 
 
 def ocrfacturas(path,task_prompt):

metrics.py

@@ -0,0 +1,150 @@
+from pydantic import BaseModel
+from anthropic import Anthropic
+import instructor
+from deepeval.models import DeepEvalBaseLLM
+from deepeval.metrics import AnswerRelevancyMetric
+from deepeval.test_case import LLMTestCase
+from deepeval.metrics import BiasMetric
+from deepeval.metrics import ToxicityMetric
+from deepeval.metrics import GEval
+from deepeval.test_case import LLMTestCaseParams
+from deepdiff import DeepDiff
+import json
+import os 
+pwd = os.getcwd()
+def extractConfig(nameModel="SystemData",relPath=os.path.join(pwd,"conf/experiment_config.json"),dataOut="keyantrophics"):
+    configPath=os.path.join(os.getcwd(),relPath)
+    with open(configPath, 'r', encoding='utf-8') as file:
+        config = json.load(file)[nameModel]
+    Output= config[dataOut]
+    return Output
+
+keyanthropic=extractConfig(nameModel="SystemData",dataOut="keyantrophics")
+
+class CustomClaudeOpus(DeepEvalBaseLLM):
+    def __init__(self):
+        self.model = Anthropic(api_key=keyanthropic)
+
+    def load_model(self):
+        return self.model
+
+    def generate(self, prompt: str, schema: BaseModel) -> BaseModel:
+        client = self.load_model()
+        instructor_client = instructor.from_anthropic(client)
+        resp = instructor_client.messages.create(
+            model="claude-3-5-sonnet-20240620",
+            max_tokens=1024,
+            messages=[
+                {
+                    "role": "user",
+                    "content": prompt,
+                }
+            ],
+            response_model=schema,
+        )
+        return resp
+
+    async def a_generate(self, prompt: str, schema: BaseModel) -> BaseModel:
+        return self.generate(prompt, schema)
+
+    def get_model_name(self):
+        return "Claude-3.5 sonnet"
+customModel=CustomClaudeOpus() 
+
+def BiasMetric22(input,actual_output):
+    metric = BiasMetric(threshold=0.5,model=customModel)
+    
+    test_case = LLMTestCase(
+        input=input,
+        actual_output=actual_output
+    )
+    metric.measure(test_case)
+    return {"score":metric.score,"reason":metric.reason}
+
+def RelevanceMetric(input,actual_output):
+# Replace this with the actual output from your LLM application
+    metric = AnswerRelevancyMetric(
+      threshold=0.7,
+      model=customModel,
+      include_reason=True
+  )
+    test_case = LLMTestCase(
+      input=input,
+      actual_output=actual_output
+  )
+
+    metric.measure(test_case)
+    return {"score":metric.score,"reason":metric.reason}
+
+
+
+
+
+def ToxicMetric(input,actual_output):
+    metric = ToxicityMetric(threshold=0.5,model=customModel)
+    test_case = LLMTestCase(
+        input=input,
+        actual_output=actual_output
+    )
+    metric.measure(test_case)
+    print(metric.score,"toxic")
+    return {"score":metric.score,"reason":metric.reason}
+
+
+
+def correctnessMetric(input,actual_output,expected_output,criteria="Determine that the output is a json whose keys contain with compra and the data correspond to the input",evaluation_steps=["Check whether the facts in 'actual output' contradicts any facts in 'expected output'","You should also heavily penalize omission of detail","Vague language, or contradicting OPINIONS, are OK" ]):
+    correctness_metric = GEval(
+        name="Correctness",
+        model=customModel,
+        criteria=criteria,
+        # NOTE: you can only provide either criteria or evaluation_steps, and not both
+        #evaluation_steps=evaluation_steps,
+        evaluation_params=[LLMTestCaseParams.INPUT, LLMTestCaseParams.ACTUAL_OUTPUT]
+    )
+    test_case = LLMTestCase(
+        input=input,
+        actual_output=actual_output,
+        expected_output=expected_output
+    )
+
+    correctness_metric.measure(test_case)
+    return {"score":correctness_metric.score,"reason":correctness_metric.reason}
+
+def jsonMetrics(text,Trusted):
+    false=False
+    print(type(text),type(Trusted))
+    try:
+        A=json.loads(text)
+        jsonOk=1
+    except:
+        jsonOk=0
+    print(jsonOk)
+    if jsonOk==1:
+
+        try:
+            Trus=json.loads(Trusted)
+        except:
+            Trus=Trusted
+    print(11111,3333,Trus)
+    #     print(type(A),type(json.loads(Trus)))
+    #     ddiff = DeepDiff(A, Trus)
+    #     print(5555,ddiff)
+    #     affectedkeys=ddiff.affected_root_keys/len(A.keys())
+    #     keys=set(json.loads(Trusted).keys())
+    #     jsonkeys=set(A.keys())
+    #     TotKey=len(keys.intersection(jsonkeys))/len(keys)
+    #     keyplus=jsonkeys.intersection(keys)
+    # else:
+    #     TotKey=0
+    #     keyplus=0
+    #     affectedkeys=0
+
+    return {"jsonOk":jsonOk}#,"TotKey":TotKey,"keyplus":keyplus,"affectedkeys":affectedkeys}
+
+
+
+
+
+
+
+

requirements.txt

@@ -2,9 +2,11 @@ aiohttp==3.9.5
 aiosignal==1.3.1
 aniso8601==9.0.1
 annotated-types==0.7.0
+anthropic==0.32.0
 anyio==4.4.0
 apispec==6.4.0
 apispec-webframeworks==1.0.0
+appdirs==1.4.4
 arrow==1.3.0
 attrs==23.2.0
 audioread==3.0.1
@@ -20,13 +22,20 @@ charset-normalizer==3.3.2
 click==8.1.7
 constantly==23.10.4
 cookiecutter==2.5.0
+dataclasses-json==0.6.7
 datasets==2.19.1
 deepdiff==6.7.1
+deepeval==0.21.74
+Deprecated==1.2.14
 dill==0.3.8
+distro==1.9.0
 dnspython==2.6.1
+docstring_parser==0.16
+docx2txt==0.8
 email_validator==2.2.0
 et-xmlfile==1.1.0
 evaluate==0.4.2
+execnet==2.1.1
 fastapi==0.111.0
 fastapi-cli==0.0.4
 filelock==3.14.0
@@ -40,7 +49,9 @@ fuzzywuzzy==0.18.0
 gevent==23.9.1
 gevent-websocket==0.10.1
 gitignore_parser==0.1.11
+googleapis-common-protos==1.63.2
 greenlet==3.0.3
+grpcio==1.63.0
 h11==0.14.0
 httpcore==1.0.5
 httptools==0.6.1
@@ -48,11 +59,23 @@ httpx==0.27.0
 huggingface-hub==0.23.2
 hyperlink==21.0.0
 idna==3.7
+importlib-metadata==7.0.0
 incremental==24.7.2
+iniconfig==2.0.0
+instructor==1.3.7
 itsdangerous==2.2.0
 Jinja2==3.1.4
+jiter==0.4.2
 jmespath==1.0.1
+jsonpatch==1.33
+jsonpointer==3.0.0
 kthread==0.2.3
+langchain==0.2.12
+langchain-community==0.2.11
+langchain-core==0.2.28
+langchain-openai==0.1.20
+langchain-text-splitters==0.2.2
+langsmith==0.1.98
 Levenshtein==0.25.1
 Markdown==3.5.2
 markdown-it-py==3.0.0
@@ -62,14 +85,26 @@ mdurl==0.1.2
 multidict==6.0.5
 multiprocess==0.70.16
 mutagen==1.47.0
+mypy-extensions==1.0.0
+nest-asyncio==1.6.0
 networkx==3.2.1
 numpy==1.26.4
+openai==1.39.0
 openpyxl==3.1.2
+opentelemetry-api==1.24.0
+opentelemetry-exporter-otlp-proto-common==1.24.0
+opentelemetry-exporter-otlp-proto-grpc==1.24.0
+opentelemetry-proto==1.24.0
+opentelemetry-sdk==1.24.0
+opentelemetry-semantic-conventions==0.45b0
 ordered-set==4.1.0
 orjson==3.10.6
 packaging==24.0
 pandas==2.2.0
 passlib==1.7.4
+pluggy==1.5.0
+portalocker==2.10.1
+protobuf==4.25.1
 pyarrow==15.0.0
 pyarrow-hotfix==0.6
 pydal==20240713.1
@@ -77,6 +112,10 @@ pydantic==2.8.2
 pydantic_core==2.20.1
 Pygments==2.18.0
 pymongo==4.6.1
+pysbd==0.3.4
+pytest==8.3.2
+pytest-repeat==0.9.3
+pytest-xdist==3.6.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-engineio==4.9.1
@@ -86,28 +125,36 @@ python-slugify==8.0.4
 python-socketio==5.11.3
 pytz==2023.3.post1
 PyYAML==6.0.1
+ragas==0.1.13
 rapidfuzz==3.9.4
+regex==2024.7.24
 requests==2.32.3
 rich==13.7.1
 s3transfer==0.10.2
+sentry-sdk==2.12.0
 shellingham==1.5.4
 simple-websocket==1.0.0
 six==1.16.0
 sniffio==1.3.1
 SQLAlchemy==2.0.25
 starlette==0.37.2
+tabulate==0.9.0
 taipy==3.1.1
 taipy-config==3.1.1
 taipy-core==3.1.1
 taipy-gui==3.1.4
 taipy-rest==3.1.1
 taipy-templates==3.1.1
+tenacity==8.4.2
 text-unidecode==1.3
+tiktoken==0.7.0
+tokenizers==0.19.1
 toml==0.10.2
 tqdm==4.66.4
 Twisted==23.10.0
 typer==0.12.3
 types-python-dateutil==2.9.0.20240316
+typing-inspect==0.9.0
 typing_extensions==4.12.0
 tzdata==2024.1
 tzlocal==5.2
@@ -118,8 +165,10 @@ uvloop==0.19.0
 watchfiles==0.22.0
 websockets==12.0
 Werkzeug==3.0.3
+wrapt==1.16.0
 wsproto==1.2.0
 xxhash==3.4.1
 yarl==1.9.4
+zipp==3.19.2
 zope.event==5.0
 zope.interface==6.4.post2