Téléverser les fichiers vers "/"
This commit is contained in:
commit
47a01d530a
370
client.py
Normal file
370
client.py
Normal file
@ -0,0 +1,370 @@
|
||||
from xml.parsers.expat import model
|
||||
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import tqdm
|
||||
import re
|
||||
from dotenv import load_dotenv
|
||||
import chromadb
|
||||
from sentence_transformers import SentenceTransformer
|
||||
import uuid
|
||||
# Load variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
# --- Configuration ---
|
||||
DUST_API_KEY = os.getenv("DUST_API_KEY")
|
||||
WORKSPACE_ID = os.getenv("DUST_WORKSPACE_ID")
|
||||
AGENT_ID = "dust" # ou l'identifiant de ton agent spécifique
|
||||
|
||||
BASE_URL = "https://dust.tt/api/v1"
|
||||
|
||||
|
||||
model = SentenceTransformer("all-MiniLM-L6-v2")
|
||||
HEADERS = {
|
||||
"Authorization": f"Bearer {DUST_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
client = chromadb.PersistentClient(path="./chroma_db")
|
||||
COLLECTION_NAME = "abap_rag"
|
||||
collection = client.get_or_create_collection(
|
||||
name=COLLECTION_NAME
|
||||
)
|
||||
|
||||
|
||||
|
||||
def create_conversation(message: str) -> dict:
|
||||
"""Crée une nouvelle conversation et envoie un premier message."""
|
||||
url = f"{BASE_URL}/w/{WORKSPACE_ID}/assistant/conversations"
|
||||
|
||||
payload = {
|
||||
"message": {
|
||||
"content": message,
|
||||
"mentions": [{"configurationId": AGENT_ID}],
|
||||
"context": {
|
||||
"timezone": "Europe/Paris",
|
||||
"username": "python_user",
|
||||
"fullName": "Python User",
|
||||
"email": "python@example.com",
|
||||
"profilePictureUrl": None,
|
||||
},
|
||||
},
|
||||
"visibility": "unlisted",
|
||||
"title": None,
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=HEADERS, json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def get_agent_message_events(conversation_id: str, message_id: str):
|
||||
"""Récupère la réponse de l'agent en streaming (SSE)."""
|
||||
url = f"{BASE_URL}/w/{WORKSPACE_ID}/assistant/conversations/{conversation_id}/messages/{message_id}/events"
|
||||
|
||||
with requests.get(url, headers=HEADERS, stream=True) as response:
|
||||
response.raise_for_status()
|
||||
full_text = ""
|
||||
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
decoded = line.decode("utf-8")
|
||||
if decoded.startswith("data:"):
|
||||
data_str = decoded[len("data:"):].strip()
|
||||
try:
|
||||
event = json.loads(data_str)
|
||||
event_type = event.get("data").get("type")
|
||||
|
||||
if event_type == "generation_tokens":
|
||||
token = event.get("data").get("text", "")
|
||||
print(token, end="", flush=True)
|
||||
full_text += token
|
||||
|
||||
elif event_type == "agent_message_success":
|
||||
print() # Saut de ligne final
|
||||
break
|
||||
|
||||
elif event_type == "agent_error":
|
||||
print(f"\n[Erreur agent] {event}")
|
||||
break
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
return full_text
|
||||
|
||||
|
||||
def send_followup_message(conversation_id: str, message: str) -> dict:
|
||||
"""Envoie un message de suivi dans une conversation existante."""
|
||||
url = f"{BASE_URL}/w/{WORKSPACE_ID}/assistant/conversations/{conversation_id}/messages"
|
||||
|
||||
payload = {
|
||||
"content": message,
|
||||
"mentions": [{"configurationId": AGENT_ID}],
|
||||
"context": {
|
||||
"timezone": "Europe/Paris",
|
||||
"username": "python_user",
|
||||
"fullName": "Python User",
|
||||
"email": "python@example.com",
|
||||
"profilePictureUrl": None,
|
||||
},
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=HEADERS, json=payload)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def chat(message: str):
|
||||
"""Point d'entrée principal : crée une conversation et affiche la réponse."""
|
||||
print(f"\n[Vous] {message}")
|
||||
print("[Agent] ", end="")
|
||||
|
||||
# 1. Créer la conversation
|
||||
result = create_conversation(message)
|
||||
conversation = result.get("conversation", {})
|
||||
conversation_id = conversation.get("sId")
|
||||
|
||||
# 2. Trouver l'ID du message agent dans la conversation
|
||||
agent_message = None
|
||||
for msg in conversation.get("content", []):
|
||||
for m in msg:
|
||||
if m.get("type") == "agent_message":
|
||||
agent_message = m
|
||||
break
|
||||
|
||||
if not agent_message:
|
||||
print("Aucun message agent trouvé.")
|
||||
return None
|
||||
|
||||
agent_message_id = agent_message.get("sId")
|
||||
|
||||
# 3. Streamer la réponse
|
||||
response_text = get_agent_message_events(conversation_id, agent_message_id)
|
||||
|
||||
return {"conversation_id": conversation_id, "response": response_text}
|
||||
|
||||
# =========================
|
||||
# INDEX FILES
|
||||
# =========================
|
||||
|
||||
def index_folder(folder_path):
|
||||
|
||||
all_files = []
|
||||
|
||||
for root, dirs, files in os.walk(folder_path):
|
||||
for file in files:
|
||||
|
||||
if file.endswith(".abap"):
|
||||
|
||||
all_files.append(
|
||||
os.path.join(root, file)
|
||||
)
|
||||
|
||||
print(f"{len(all_files)} fichiers trouvés")
|
||||
|
||||
for filepath in tqdm.tqdm(all_files):
|
||||
|
||||
try:
|
||||
|
||||
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
|
||||
content = f.read()
|
||||
|
||||
chunks = split_abap_code(content)
|
||||
|
||||
for chunk in chunks:
|
||||
|
||||
embedding = model.encode(chunk).tolist()
|
||||
|
||||
collection.add(
|
||||
ids=[str(uuid.uuid4())],
|
||||
documents=[chunk],
|
||||
embeddings=[embedding],
|
||||
metadatas=[{
|
||||
"source": filepath,
|
||||
"filename": os.path.basename(filepath)
|
||||
}]
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Erreur indexation {filepath}: {e}")
|
||||
|
||||
print("Indexation terminée")
|
||||
# =========================
|
||||
# DOCUMENT FILE WITH RAG
|
||||
# =========================
|
||||
|
||||
def document_file(filepath):
|
||||
|
||||
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
|
||||
content = f.read()
|
||||
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
# recherche RAG
|
||||
rag_context = retrieve_context(content, top_k=5)
|
||||
|
||||
prompt = f"""
|
||||
Tu es un expert SAP ABAP.
|
||||
|
||||
Tu dois documenter le fichier suivant.
|
||||
|
||||
FICHIER ANALYSÉ:
|
||||
{filename}
|
||||
|
||||
CHEMIN:
|
||||
{filepath}
|
||||
|
||||
CODE PRINCIPAL:
|
||||
{content}
|
||||
|
||||
CONTEXTE RAG:
|
||||
{rag_context}
|
||||
|
||||
Ton résultat doit être en markdown avec plusieurs sections :
|
||||
|
||||
# Algorithmie
|
||||
Explique le fonctionnement général du programme.
|
||||
|
||||
# Objets
|
||||
Explique les objets utilisés et leur rôle.
|
||||
|
||||
# Dépendances
|
||||
Explique les dépendances importantes.
|
||||
|
||||
# Flux de traitement
|
||||
Décris les principales étapes d'exécution.
|
||||
|
||||
# Points d'attention
|
||||
Liste les risques techniques ou métiers.
|
||||
"""
|
||||
|
||||
return chat(prompt)
|
||||
# =========================
|
||||
# ABAP CHUNKER
|
||||
# =========================
|
||||
|
||||
def split_abap_code(content):
|
||||
"""
|
||||
Découpe :
|
||||
- METHOD
|
||||
- FORM
|
||||
- FUNCTION
|
||||
"""
|
||||
|
||||
patterns = [
|
||||
r"METHOD\s+.*?ENDMETHOD\.",
|
||||
r"FORM\s+.*?ENDFORM\.",
|
||||
r"FUNCTION\s+.*?ENDFUNCTION\."
|
||||
]
|
||||
|
||||
chunks = []
|
||||
|
||||
for pattern in patterns:
|
||||
matches = re.findall(
|
||||
pattern,
|
||||
content,
|
||||
re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
|
||||
chunks.extend(matches)
|
||||
|
||||
# fallback si aucun chunk
|
||||
if not chunks:
|
||||
chunks = [content]
|
||||
|
||||
return chunks
|
||||
# =========================
|
||||
# RAG SEARCH
|
||||
# =========================
|
||||
|
||||
def retrieve_context(query, top_k=5):
|
||||
|
||||
query_embedding = model.encode(query).tolist()
|
||||
|
||||
results = collection.query(
|
||||
query_embeddings=[query_embedding],
|
||||
n_results=top_k
|
||||
)
|
||||
|
||||
documents = results["documents"][0]
|
||||
metadatas = results["metadatas"][0]
|
||||
|
||||
context_parts = []
|
||||
|
||||
for doc, meta in zip(documents, metadatas):
|
||||
|
||||
source = meta.get("source", "unknown")
|
||||
|
||||
context_parts.append(f"""
|
||||
SOURCE FILE:
|
||||
{source}
|
||||
|
||||
CODE:
|
||||
{doc}
|
||||
""")
|
||||
|
||||
return "\n\n".join(context_parts)
|
||||
# --- Programme principal ---
|
||||
if __name__ == "__main__":
|
||||
folder_or_file = input("Entrez le chemin d'un dossier ou d'un fichier à analyser : ")
|
||||
outputdir = input("Entrez le chemin du dossier de sortie pour les documentations générées : ")
|
||||
if os.path.isdir(folder_or_file):
|
||||
all_files = []
|
||||
|
||||
for root, dirs, files in os.walk(folder_or_file):
|
||||
for file in files:
|
||||
if file.endswith(".abap"):
|
||||
all_files.append(
|
||||
os.path.join(root, file)
|
||||
)
|
||||
print("INDEXATION RAG...")
|
||||
index_folder(folder_or_file)
|
||||
|
||||
print("GÉNÉRATION DOCUMENTATION...")
|
||||
|
||||
for root, dirs, files in os.walk(folder_or_file):
|
||||
|
||||
for file in files:
|
||||
|
||||
if file.endswith(".abap"):
|
||||
|
||||
filepath = os.path.join(root, file)
|
||||
|
||||
print(f"\nDocumentation de {filepath}")
|
||||
|
||||
result = document_file(filepath)
|
||||
|
||||
if result:
|
||||
|
||||
documentation = result['response']
|
||||
|
||||
output_name = os.path.splitext(file)[0]
|
||||
|
||||
with open(
|
||||
f"{outputdir}/documentation_{output_name}.md",
|
||||
"w",
|
||||
encoding="utf-8"
|
||||
) as f:
|
||||
|
||||
f.write(documentation)
|
||||
|
||||
print(f"Documentation générée : documentation_{output_name}.md")
|
||||
elif os.path.isfile(folder_or_file):
|
||||
content = open(folder_or_file, "r").read()
|
||||
result = chat(f'''peux-tu m'expliquer le fonctionnement du fichier {content} ton résultat devra être en markdown avec plusieurs sections :
|
||||
une section algorithmie, où tu explique le fonctionnement général du programme
|
||||
une section objets, où tu explique les différents objets utilisés dans le programme et leur rôle.''' )
|
||||
|
||||
if result:
|
||||
print(f"\n[réponse: {result}]")
|
||||
print(f"\n[Conversation ID: {result['conversation_id']}]")
|
||||
documentation = result['response']
|
||||
file_name = os.path.basename(folder_or_file).split('.')[0]
|
||||
with open(f"documentation_{file_name}.md", "w", encoding="utf-8") as f:
|
||||
f.write(documentation)
|
||||
else:
|
||||
print("Le chemin spécifié n'est ni un fichier ni un dossier valide.")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user