feat: fix hotspot test

2026-01-22 16:52:36 -03:00
parent 957539d201
commit fa8ca6cb61
20 changed files with 247 additions and 39 deletions
@@ -27,7 +27,7 @@ El proyecto ha sido optimizado para reducir la complejidad de la infraestructura
    - **Custom AI Quizzes**: Generación de quices con contexto pedagógico y tipo de pregunta personalizable (opción múltiple, V/F, etc.).
    - **Course Deletion**: Funcionalidad de eliminación de cursos con verificación de permisos y limpieza en cascada.
    - **Gamified Activities**: Nuevos tipos de bloques interactivos para niños y jóvenes, incluyendo Juegos de Memoria y Puntos Calientes (Hotspots).
-    - **Gamified Activities**: Nuevos tipos de bloques interactivos para niños y jóvenes, incluyendo Juegos de Memoria y Puntos Calientes (Hotspots).
+    - **Auto Transcription**: Integración con Whisper para generación automática de transcripciones y evaluación precisa de voz.

 ##  Requisitos del Sistema

@@ -42,6 +42,8 @@ services:
      JWT_SECRET: openccb_secret_key_2025_production
      NEXT_PUBLIC_LMS_API_URL: http://localhost:3002
    env_file: .env
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
    depends_on:
      - db

@@ -576,9 +576,29 @@ pub async fn create_lesson(
        .await
        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;

+    // Trigger auto-transcription if it's a video or audio lesson with a URL
+    if (content_type == "video" || content_type == "audio") && content_url.is_some() {
+        trigger_transcription(pool, lesson.id).await;
+    }
+
    Ok(Json(lesson))
 }

+async fn trigger_transcription(pool: PgPool, lesson_id: Uuid) {
+    // Set status to queued
+    let _ = sqlx::query("UPDATE lessons SET transcription_status = 'queued' WHERE id = $1")
+        .bind(lesson_id)
+        .execute(&pool)
+        .await;
+
+    // Spawn background task
+    tokio::spawn(async move {
+        if let Err(e) = run_transcription_task(pool, lesson_id).await {
+            tracing::error!("Auto-transcription task failed for lesson {}: {}", lesson_id, e);
+        }
+    });
+}
+
 pub async fn process_transcription(
    Org(org_ctx): Org,
    claims: common::auth::Claims,
@@ -646,7 +666,7 @@ pub async fn process_transcription(
    Ok(Json(updated_lesson))
 }

-async fn translate_text(text: &str, target_lang: &str) -> Result<String, String> {
+async fn _translate_text(text: &str, target_lang: &str) -> Result<String, String> {
    let provider = env::var("AI_PROVIDER").unwrap_or_else(|_| "openai".to_string());
    let client = reqwest::Client::new();

@@ -1286,6 +1306,16 @@ pub async fn update_lesson(
        .await
        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;

+    // Trigger auto-transcription if content URL was updated and it's a video/audio lesson
+    if let Some(url) = content_url {
+        if !url.is_empty() {
+            let c_type = content_type.unwrap_or(lesson.content_type.as_str());
+            if c_type == "video" || c_type == "audio" {
+                trigger_transcription(pool, lesson.id).await;
+            }
+        }
+    }
+
    Ok(Json(lesson))
 }

@@ -0,0 +1,4 @@
+-- Migration: Add updated_at to user_grades
+-- Required by fn_upsert_user_grade matching CMS-style upserts
+
+ALTER TABLE user_grades ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
@@ -1,6 +1,6 @@
 use axum::{
    Json,
-    extract::{Path, Query, State},
+    extract::{Path, Query, State, Multipart},
    http::StatusCode,
 };
 use bcrypt::{DEFAULT_COST, hash, verify};
@@ -991,8 +991,9 @@ pub async fn check_deadlines_and_notify(pool: PgPool) {
            'deadline',
            '/courses/' || c.id || '/lessons/' || l.id
         FROM enrollments e
-         JOIN lessons l ON l.course_id = e.course_id
-         JOIN courses c ON c.id = l.course_id
+         JOIN courses c ON c.id = e.course_id
+         JOIN modules m ON m.course_id = c.id
+         JOIN lessons l ON l.module_id = m.id
         WHERE l.due_date BETWEEN NOW() AND NOW() + INTERVAL '24 hours'
         AND NOT EXISTS (
            SELECT 1 FROM notifications n 
@@ -1054,7 +1055,7 @@ pub async fn update_user(
 }

 pub async fn get_recommendations(
-    Org(org_ctx): Org,
+    Org(_org_ctx): Org,
    claims: Claims,
    State(pool): State<PgPool>,
    Path(course_id): Path<Uuid>,
@@ -1221,3 +1222,126 @@ pub async fn evaluate_audio_response(

    Ok(Json(grading))
 }
+
+pub async fn evaluate_audio_file(
+    Org(_org_ctx): Org,
+    _claims: Claims,
+    mut multipart: Multipart,
+) -> Result<Json<AudioGradingResponse>, (StatusCode, String)> {
+    let mut prompt = String::new();
+    let mut keywords_str = String::new();
+    let mut audio_data = Vec::new();
+    let mut filename = "audio.webm".to_string();
+
+    while let Some(field) = multipart.next_field().await.map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))? {
+        let name = field.name().unwrap_or_default().to_string();
+        match name.as_str() {
+            "prompt" => prompt = field.text().await.unwrap_or_default(),
+            "keywords" => keywords_str = field.text().await.unwrap_or_default(),
+            "file" => {
+                filename = field.file_name().unwrap_or("audio.webm").to_string();
+                audio_data = field.bytes().await.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?.to_vec();
+            },
+            _ => {}
+        }
+    }
+
+    if audio_data.is_empty() {
+        return Err((StatusCode::BAD_REQUEST, "No audio file provided".into()));
+    }
+
+    // 1. Send to Whisper
+    let whisper_url = env::var("LOCAL_WHISPER_URL").unwrap_or_else(|_| "http://localhost:8000".to_string());
+    let client = reqwest::Client::new();
+    
+    let form = reqwest::multipart::Form::new()
+        .part("file", reqwest::multipart::Part::bytes(audio_data).file_name(filename))
+        .text("model", "whisper-1")
+        .text("response_format", "json");
+
+    let response = client.post(format!("{}/v1/audio/transcriptions", whisper_url))
+        .multipart(form)
+        .send()
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Whisper request failed: {}", e)))?;
+
+    if !response.status().is_success() {
+        let err_body = response.text().await.unwrap_or_default();
+        tracing::error!("Whisper error: {}", err_body);
+        return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("Whisper API error: {}", err_body)));
+    }
+
+    let transcription_result: serde_json::Value = response.json().await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to parse Whisper response: {}", e)))?;
+
+    let transcript = transcription_result["text"].as_str().unwrap_or("").to_string();
+    
+    if transcript.is_empty() {
+        return Err((StatusCode::BAD_REQUEST, "Whisper could not detect any speech. Please speak louder or check your mic.".into()));
+    }
+
+    let keywords: Vec<String> = if keywords_str.trim().starts_with('[') {
+        serde_json::from_str(&keywords_str).unwrap_or_default()
+    } else {
+        keywords_str.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect()
+    };
+
+    // 2. Perform AI Grading
+    let provider = env::var("AI_PROVIDER").unwrap_or_else(|_| "openai".to_string());
+    let (url, auth_header, model) = if provider == "local" {
+        let base_url = env::var("LOCAL_OLLAMA_URL").unwrap_or_else(|_| "http://ollama:11434".to_string());
+        let model = env::var("LOCAL_LLM_MODEL").unwrap_or_else(|_| "llama3:8b".to_string());
+        (format!("{}/v1/chat/completions", base_url), "".to_string(), model)
+    } else {
+        (
+            "https://api.openai.com/v1/chat/completions".to_string(),
+            format!("Bearer {}", env::var("OPENAI_API_KEY").unwrap_or_default()),
+            "gpt-4-turbo".to_string(),
+        )
+    };
+
+    let system_prompt = "You are an expert Teacher. Evaluate the student's spoken response transcript. \
+        Compare it against the prompt and expected keywords. \
+        Provide a score from 0 to 100. \
+        Identify which keywords were used. \
+        Give constructive feedback in Spanish about their pronunciation (based on the transcript quality) and content. \
+        Return ONLY a JSON object: { \"score\": number, \"found_keywords\": [string], \"feedback\": string }.";
+
+    let user_content = format!(
+        "Prompt: {}\nExpected Keywords: {:?}\nStudent Transcript: {}",
+        prompt, keywords, transcript
+    );
+
+    let response = client.post(&url)
+        .header("Content-Type", "application/json")
+        .header("Authorization", auth_header)
+        .json(&serde_json::json!({
+            "model": model,
+            "messages": [
+                { "role": "system", "content": system_prompt },
+                { "role": "user", "content": user_content }
+            ],
+            "response_format": { "type": "json_object" }
+        }))
+        .send()
+        .await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
+
+    let ai_data: serde_json::Value = response.json().await
+        .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("AI response parse failed: {}", e)))?;
+
+    let grading: AudioGradingResponse = serde_json::from_value(
+        ai_data["choices"][0]["message"]["content"]
+            .as_str()
+            .and_then(|c| serde_json::from_str(c).ok())
+            .unwrap_or_else(|| {
+                serde_json::json!({
+                    "score": 50,
+                    "found_keywords": vec![] as Vec<String>,
+                    "feedback": "Lo siento, tuve un problema analizando tu respuesta con Whisper. ¡Sigue practicando!"
+                })
+            })
+    ).map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("Mapping failed: {}", e)))?;
+
+    Ok(Json(grading))
+}
@@ -80,6 +80,7 @@ async fn main() {
            get(handlers::get_lesson_heatmap),
        )
        .route("/audio/evaluate", post(handlers::evaluate_audio_response))
+        .route("/audio/evaluate-file", post(handlers::evaluate_audio_file))
        .route("/notifications", get(handlers::get_notifications))
        .route(
            "/notifications/{id}/read",
@@ -1,6 +1,16 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
    output: "standalone",
+    images: {
+        remotePatterns: [
+            {
+                protocol: 'http',
+                hostname: 'localhost',
+                port: '3001',
+                pathname: '/assets/**',
+            },
+        ],
+    },
 };

 export default nextConfig;
@@ -22,12 +22,12 @@ export default function StudentCalendarPage({ params }: { params: { id: string }
    useEffect(() => {
        const loadData = async () => {
            try {
-                const courseData = await lmsApi.getCourseOutline(params.id);
-                setCourse(courseData);
+                const { course, modules } = await lmsApi.getCourseOutline(params.id);
+                setCourse({ ...course, modules });

                // Flatten lessons from modules
                const allLessons: Lesson[] = [];
-                courseData.modules?.forEach(mod => {
+                modules?.forEach(mod => {
                    mod.lessons.forEach(lesson => {
                        allLessons.push(lesson);
                    });
@@ -34,12 +34,12 @@ export default function LessonPlayerPage({ params }: { params: { id: string, les
    useEffect(() => {
        const fetchAll = async () => {
            try {
-                const [lessonData, courseData] = await Promise.all([
+                const [lessonData, outlineData] = await Promise.all([
                    lmsApi.getLesson(params.lessonId),
                    lmsApi.getCourseOutline(params.id)
                ]);
                setLesson(lessonData);
-                setCourse(courseData);
+                setCourse({ ...outlineData.course, modules: outlineData.modules });

                if (user) {
                    const grades = await lmsApi.getUserGrades(user.id, params.id);
@@ -84,11 +84,25 @@ export default function LessonPlayerPage({ params }: { params: { id: string, les
                    [blockId]: score
                };

+                // Calculate overall lesson score as average of block scores
+                const blocks = lesson.metadata?.blocks || [];
+                const interactiveBlocks = blocks.filter((b: any) =>
+                    !['description', 'media', 'document'].includes(b.type)
+                );
+
+                const scores = interactiveBlocks.map((b: any) =>
+                    b.id === blockId ? score : currentBlockScores[b.id] || 0
+                );
+
+                const newOverallScore = scores.length > 0
+                    ? (scores.reduce((a: number, b: number) => a + b, 0) / scores.length) * 100
+                    : 100;
+
                const res = await lmsApi.submitScore(
                    user.id,
                    params.id,
                    params.lessonId,
-                    userGrade?.score || 0, // Keep overall score for now, or calculate average/sum
+                    newOverallScore,
                    { ...userGrade?.metadata, block_scores: newBlockScores }
                );
                setUserGrade(res);
@@ -220,6 +234,7 @@ export default function LessonPlayerPage({ params }: { params: { id: string, les
                                                                }
                                                            }}
                                                            isGraded={lesson.is_graded}
+                                                            hasTranscription={!!lesson.transcription}
                                                        />
                                                    );
                                                case 'document':
@@ -14,7 +14,7 @@ export default function CourseOutlinePage({ params }: { params: { id: string } }

    useEffect(() => {
        lmsApi.getCourseOutline(params.id)
-            .then(setCourseData)
+            .then(data => setCourseData({ ...data.course, modules: data.modules }))
            .catch(console.error)
            .finally(() => setLoading(false));

@@ -32,8 +32,8 @@ export default function StudentProgressPage() {

    const loadData = React.useCallback(async () => {
        try {
-            const courseData = await lmsApi.getCourseOutline(id);
-            setCourse(courseData);
+            const { course, modules, grading_categories } = await lmsApi.getCourseOutline(id);
+            setCourse({ ...course, modules, grading_categories });

            if (user) {
                const grades = await lmsApi.getUserGrades(user.id, id);
@@ -34,7 +34,7 @@ export default function RootLayout({
                </main>
                <footer className="py-12 px-6 border-t border-white/5 text-center bg-black/20">
                  <p className="text-[10px] font-black uppercase tracking-[0.2em] text-gray-600">
-                    Desarrollado por OpenCCB © 2023. Codificación Agente Avanzada.
+                    Desarrollado Por el Departamento de Informática © 2026. OpenCCB.
                  </p>
                </footer>
              </AuthGuard>
@@ -38,14 +38,14 @@ export default function MyLearningPage() {
                const enrichedEnrollments: EnrollmentWithProgress[] = [];
                for (const enrollment of enrollmentData) {
                    try {
-                        const outline = await lmsApi.getCourseOutline(enrollment.course_id);
+                        const { course, modules } = await lmsApi.getCourseOutline(enrollment.course_id);

                        // TODO: Implement actual progress tracking
                        // For now, show 0% progress for all courses
                        const progress = 0;

                        enrichedEnrollments.push({
-                            course: outline,
+                            course: { ...course, modules },
                            progress,
                            lastAccessed: enrollment.enroled_at
                        });
@@ -35,11 +35,11 @@ export default function CatalogPage() {
          const deadlines: { lesson: Lesson, courseTitle: string, courseId: string }[] = [];
          for (const enrollment of enrollmentData) {
            try {
-              const outline = await lmsApi.getCourseOutline(enrollment.course_id);
-              outline.modules.forEach(mod => {
+              const { course, modules } = await lmsApi.getCourseOutline(enrollment.course_id);
+              modules.forEach(mod => {
                mod.lessons.forEach(l => {
                  if (l.due_date && new Date(l.due_date) >= new Date()) {
-                    deadlines.push({ lesson: l, courseTitle: outline.title, courseId: enrollment.course_id });
+                    deadlines.push({ lesson: l, courseTitle: course.title, courseId: enrollment.course_id });
                  }
                });
              });
@@ -135,14 +135,14 @@ export default function AudioResponsePlayer({
    };

    const evaluateResponse = async () => {
-        if (!transcript.trim()) {
-            alert("No speech detected. Please try recording again.");
+        if (!audioBlob) {
+            alert("No recording found. Please try again.");
            return;
        }

        setIsTranscribing(true);
        try {
-            const result = await lmsApi.evaluateAudio(transcript, prompt, keywords);
+            const result = await lmsApi.evaluateAudioFile(audioBlob, prompt, keywords);
            setEvaluation({
                score: result.score,
                foundKeywords: result.found_keywords,
@@ -153,9 +153,9 @@ export default function AudioResponsePlayer({
            if (onComplete) {
                onComplete(result.score, transcript);
            }
-        } catch (err) {
+        } catch (err: any) {
            console.error("Evaluation failed", err);
-            alert("Evaluation failed. Please try again.");
+            alert(err.message || "Evaluation failed. Please try again.");
        } finally {
            setIsTranscribing(false);
        }
@@ -94,6 +94,7 @@ export default function HotspotPlayer({
                    src={getImageUrl(imageUrl)}
                    alt={title}
                    fill
+                    unoptimized
                    className="object-cover transition-transform duration-700 group-hover:scale-[1.02]"
                    sizes="(max-width: 768px) 100vw, (max-width: 1200px) 50vw, 33vw"
                />
@@ -12,6 +12,7 @@ interface MediaPlayerProps {
    media_type: 'video' | 'audio';
    config?: {
        maxPlays?: number;
+        show_transcript?: boolean;
        markers?: {
            timestamp: number;
            question: string;
@@ -176,10 +177,10 @@ export default function MediaPlayer({ id, lessonId, title, url, media_type, conf
                            }
                        }}
                    >
-                        {hasTranscription && vttEn && (
+                        {hasTranscription && config?.show_transcript !== false && vttEn && (
                            <track kind="subtitles" src={vttEn} srcLang="en" label="English" />
                        )}
-                        {hasTranscription && vttEs && (
+                        {hasTranscription && config?.show_transcript !== false && vttEs && (
                            <track kind="subtitles" src={vttEs} srcLang="es" label="Español" />
                        )}
                    </video>
@@ -209,7 +209,7 @@ export const lmsApi = {
        return apiFetch(`/catalog${query}`);
    },

-    async getCourseOutline(courseId: string): Promise<Course & { modules: Module[], grading_categories: GradingCategory[] }> {
+    async getCourseOutline(courseId: string): Promise<{ course: Course, modules: Module[], grading_categories: GradingCategory[], organization: Organization }> {
        return apiFetch(`/courses/${courseId}/outline`);
    },

@@ -321,5 +321,26 @@ export const lmsApi = {
            method: 'POST',
            body: JSON.stringify({ transcript, prompt, keywords })
        });
+    },
+    async evaluateAudioFile(file: Blob, prompt: string, keywords: string[]): Promise<AudioGradingResponse> {
+        const formData = new FormData();
+        formData.append('file', file, 'recorded_audio.webm');
+        formData.append('prompt', prompt);
+        formData.append('keywords', JSON.stringify(keywords));
+
+        const token = getToken();
+        return fetch(`${API_BASE_URL}/audio/evaluate-file`, {
+            method: 'POST',
+            headers: {
+                ...(token ? { 'Authorization': `Bearer ${token}` } : {})
+            },
+            body: formData
+        }).then(async res => {
+            if (!res.ok) {
+                const err = await res.json().catch(() => ({ message: 'Audio evaluation failed' }));
+                throw new Error(err.message || 'Audio evaluation failed');
+            }
+            return res.json();
+        });
    }
 };
@@ -87,7 +87,7 @@ export default function HotspotBlock({
                </div>
                <div className="relative aspect-video rounded-2xl overflow-hidden border border-white/5 bg-black/40">
                    {imageUrl ? (
-                        <Image src={getImageUrl(imageUrl)} alt={title || ""} fill className="object-cover opacity-50" />
+                        <Image src={getImageUrl(imageUrl)} alt={title || ""} fill unoptimized className="object-cover opacity-50" />
                    ) : (
                        <div className="absolute inset-0 flex items-center justify-center text-gray-600 italic text-sm">No image provided.</div>
                    )}
@@ -140,7 +140,7 @@ export default function HotspotBlock({
                            </button>
                        ) : (
                            <div className="relative aspect-video rounded-2xl overflow-hidden group">
-                                <Image src={getImageUrl(imageUrl)} alt="Hotspot base" fill className="object-cover" />
+                                <Image src={getImageUrl(imageUrl)} alt="Hotspot base" fill unoptimized className="object-cover" />
                                <div className="absolute inset-0 bg-black/60 opacity-0 group-hover:opacity-100 transition-opacity flex items-center justify-center gap-3">
                                    <button onClick={() => setIsAssetPickerOpen(true)} className="p-2 bg-white/10 hover:bg-white/20 rounded-lg text-white transition-all"><ImageIcon size={18} /></button>
                                    <button onClick={() => onChange({ imageUrl: undefined })} className="p-2 bg-red-500/20 hover:bg-red-500/40 rounded-lg text-red-400 transition-all"><Trash2 size={18} /></button>
@@ -164,8 +164,8 @@ export default function HotspotBlock({
                                    onClick={handleImageClick}
                                    className="relative aspect-video rounded-2xl overflow-hidden border-2 border-white/10 cursor-crosshair shadow-2xl"
                                >
-                                    <Image src={getImageUrl(imageUrl)} alt="Define Hotspots" fill className="object-cover select-none" />
-                                    {hotspots.map((h, idx) => (
+                                    <Image src={getImageUrl(imageUrl)} alt="Define Hotspots" fill unoptimized className="object-cover select-none" />
+                                    {hotspots.map((h) => (
                                        <div
                                            key={h.id}
                                            className="absolute group/pin"
@@ -246,6 +246,6 @@ export default function HotspotBlock({
                filterType="image"
                onSelect={handleImageSelect}
            />
-        </div>
+        </div >
    );
 }
@@ -1,6 +1,5 @@
 "use client";

-import { useState } from "react";
 import { Brain, Plus, Trash2, HelpCircle } from "lucide-react";

 interface MatchingPair {