feat: Implement AI-powered audio response evaluation with score, keywords, and feedback, integrating it into the AudioResponsePlayer component.
This commit is contained in:
@@ -16,7 +16,8 @@ import ShortAnswerPlayer from "@/components/blocks/ShortAnswerPlayer";
|
||||
import CodeExercisePlayer from "@/components/blocks/CodeExercisePlayer";
|
||||
import HotspotPlayer from "@/components/blocks/HotspotPlayer";
|
||||
import MemoryPlayer from "@/components/blocks/MemoryPlayer";
|
||||
import DocumentPlayer from "@/components/blocks/DocumentPlayer"; // Added import
|
||||
import DocumentPlayer from "@/components/blocks/DocumentPlayer";
|
||||
import AudioResponsePlayer from "@/components/blocks/AudioResponsePlayer";
|
||||
import InteractiveTranscript from "@/components/InteractiveTranscript";
|
||||
import { ListMusic } from "lucide-react";
|
||||
|
||||
@@ -276,6 +277,17 @@ export default function LessonPlayerPage({ params }: { params: { id: string, les
|
||||
allowRetry={lesson.allow_retry}
|
||||
/>
|
||||
);
|
||||
case 'audio-response':
|
||||
return (
|
||||
<AudioResponsePlayer
|
||||
id={block.id}
|
||||
prompt={block.prompt || ""}
|
||||
keywords={block.keywords}
|
||||
timeLimit={block.timeLimit}
|
||||
isGraded={lesson.is_graded}
|
||||
onComplete={(score) => handleBlockComplete(block.id, score)}
|
||||
/>
|
||||
);
|
||||
case 'code':
|
||||
return (
|
||||
<CodeExercisePlayer
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useRef, useEffect } from "react";
|
||||
import { Mic, Square, Play, RotateCcw, Check, X, Clock } from "lucide-react";
|
||||
import { lmsApi } from "@/lib/api";
|
||||
import { Mic, Square, Play, RotateCcw, Check, X, Clock, BrainCircuit } from "lucide-react";
|
||||
|
||||
interface AudioResponsePlayerProps {
|
||||
id: string;
|
||||
@@ -26,7 +27,7 @@ export default function AudioResponsePlayer({
|
||||
const [isTranscribing, setIsTranscribing] = useState(false);
|
||||
const [recordingTime, setRecordingTime] = useState(0);
|
||||
const [submitted, setSubmitted] = useState(false);
|
||||
const [evaluation, setEvaluation] = useState<{ score: number; foundKeywords: string[] } | null>(null);
|
||||
const [evaluation, setEvaluation] = useState<{ score: number; foundKeywords: string[]; feedback: string } | null>(null);
|
||||
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
||||
const audioChunksRef = useRef<Blob[]>([]);
|
||||
@@ -133,26 +134,30 @@ export default function AudioResponsePlayer({
|
||||
}
|
||||
};
|
||||
|
||||
const evaluateResponse = () => {
|
||||
const evaluateResponse = async () => {
|
||||
if (!transcript.trim()) {
|
||||
alert("No speech detected. Please try recording again.");
|
||||
return;
|
||||
}
|
||||
|
||||
const transcriptLower = transcript.toLowerCase();
|
||||
const foundKeywords = keywords.filter(kw =>
|
||||
transcriptLower.includes(kw.toLowerCase())
|
||||
);
|
||||
setIsTranscribing(true);
|
||||
try {
|
||||
const result = await lmsApi.evaluateAudio(transcript, prompt, keywords);
|
||||
setEvaluation({
|
||||
score: result.score,
|
||||
foundKeywords: result.found_keywords,
|
||||
feedback: result.feedback
|
||||
});
|
||||
setSubmitted(true);
|
||||
|
||||
const score = keywords.length > 0
|
||||
? Math.round((foundKeywords.length / keywords.length) * 100)
|
||||
: 100; // If no keywords specified, give full credit for any response
|
||||
|
||||
setEvaluation({ score, foundKeywords });
|
||||
setSubmitted(true);
|
||||
|
||||
if (onComplete) {
|
||||
onComplete(score, transcript);
|
||||
if (onComplete) {
|
||||
onComplete(result.score, transcript);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Evaluation failed", err);
|
||||
alert("Evaluation failed. Please try again.");
|
||||
} finally {
|
||||
setIsTranscribing(false);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -269,10 +274,10 @@ export default function AudioResponsePlayer({
|
||||
{submitted && evaluation && (
|
||||
<div className="space-y-4">
|
||||
<div className={`p-6 rounded-2xl border-2 ${evaluation.score >= 70
|
||||
? 'bg-green-500/10 border-green-500'
|
||||
: evaluation.score >= 40
|
||||
? 'bg-yellow-500/10 border-yellow-500'
|
||||
: 'bg-red-500/10 border-red-500'
|
||||
? 'bg-green-500/10 border-green-500'
|
||||
: evaluation.score >= 40
|
||||
? 'bg-yellow-500/10 border-yellow-500'
|
||||
: 'bg-red-500/10 border-red-500'
|
||||
}`}>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<span className="text-sm font-bold uppercase tracking-wider text-gray-400">Your Score</span>
|
||||
@@ -289,8 +294,8 @@ export default function AudioResponsePlayer({
|
||||
<span
|
||||
key={i}
|
||||
className={`px-3 py-1 rounded-lg text-sm font-medium flex items-center gap-1 ${found
|
||||
? 'bg-green-500/20 border border-green-500/50 text-green-300'
|
||||
: 'bg-gray-500/20 border border-gray-500/50 text-gray-400'
|
||||
? 'bg-green-500/20 border border-green-500/50 text-green-300'
|
||||
: 'bg-gray-500/20 border border-gray-500/50 text-gray-400'
|
||||
}`}
|
||||
>
|
||||
{found ? <Check className="w-3 h-3" /> : <X className="w-3 h-3" />}
|
||||
@@ -303,6 +308,21 @@ export default function AudioResponsePlayer({
|
||||
)}
|
||||
</div>
|
||||
|
||||
{evaluation.feedback && (
|
||||
<div className="p-6 bg-blue-500/10 border-2 border-blue-500/20 rounded-2xl space-y-3 relative overflow-hidden group">
|
||||
<div className="absolute top-0 right-0 p-4 opacity-5 group-hover:opacity-10 transition-opacity">
|
||||
<BrainCircuit className="w-20 h-20 text-blue-400" />
|
||||
</div>
|
||||
<div className="flex items-center gap-2 text-xs font-black uppercase tracking-widest text-blue-400">
|
||||
<BrainCircuit className="w-4 h-4" />
|
||||
AI Teacher Feedback
|
||||
</div>
|
||||
<p className="text-gray-200 leading-relaxed italic text-lg relative z-10">
|
||||
"{evaluation.feedback}"
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="p-4 bg-white/5 border border-white/10 rounded-xl">
|
||||
<p className="text-xs text-gray-500 uppercase tracking-wider mb-2">Your Transcript:</p>
|
||||
<p className="text-sm text-gray-300">{transcript}</p>
|
||||
|
||||
@@ -28,6 +28,12 @@ export interface RecommendationResponse {
|
||||
recommendations: Recommendation[];
|
||||
}
|
||||
|
||||
export interface AudioGradingResponse {
|
||||
score: number;
|
||||
found_keywords: string[];
|
||||
feedback: string;
|
||||
}
|
||||
|
||||
export interface Course {
|
||||
id: string;
|
||||
title: string;
|
||||
@@ -51,7 +57,7 @@ export interface QuizQuestion {
|
||||
|
||||
export interface Block {
|
||||
id: string;
|
||||
type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'code' | 'hotspot' | 'memory-match' | 'document';
|
||||
type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'code' | 'hotspot' | 'memory-match' | 'document' | 'audio-response' | 'video_marker';
|
||||
title: string;
|
||||
content?: string;
|
||||
url?: string;
|
||||
@@ -66,6 +72,8 @@ export interface Block {
|
||||
correctAnswers?: string[];
|
||||
instructions?: string;
|
||||
initialCode?: string;
|
||||
keywords?: string[];
|
||||
timeLimit?: number;
|
||||
metadata?: any;
|
||||
}
|
||||
|
||||
@@ -298,5 +306,11 @@ export const lmsApi = {
|
||||
},
|
||||
async getRecommendations(courseId: string): Promise<RecommendationResponse> {
|
||||
return apiFetch(`/courses/${courseId}/recommendations`);
|
||||
},
|
||||
async evaluateAudio(transcript: string, prompt: string, keywords: string[]): Promise<AudioGradingResponse> {
|
||||
return apiFetch('/audio/evaluate', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ transcript, prompt, keywords })
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
@@ -12,6 +12,7 @@ import OrderingBlock from "@/components/blocks/OrderingBlock";
|
||||
import ShortAnswerBlock from "@/components/blocks/ShortAnswerBlock";
|
||||
import DocumentBlock from "@/components/blocks/DocumentBlock";
|
||||
import VideoMarkerBlock from "@/components/blocks/VideoMarkerBlock";
|
||||
import AudioResponseBlock from "@/components/blocks/AudioResponseBlock";
|
||||
import {
|
||||
Save,
|
||||
X,
|
||||
@@ -155,7 +156,7 @@ export default function LessonEditor({ params }: { params: { id: string; lessonI
|
||||
}
|
||||
};
|
||||
|
||||
const addBlock = (type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'document' | 'video_marker') => {
|
||||
const addBlock = (type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'document' | 'video_marker' | 'audio-response') => {
|
||||
const newBlock: Block = {
|
||||
id: Math.random().toString(36).substr(2, 9),
|
||||
type,
|
||||
@@ -168,6 +169,7 @@ export default function LessonEditor({ params }: { params: { id: string; lessonI
|
||||
...(type === 'short-answer' && { prompt: "Question?", correctAnswers: ["Answer"] }),
|
||||
...(type === 'document' && { url: "", title: "" }),
|
||||
...(type === 'video_marker' && { url: "", title: "Video Interactivo", markers: [] }),
|
||||
...(type === 'audio-response' && { prompt: "Ask a question for the student to record their answer...", keywords: [], timeLimit: 60 }),
|
||||
};
|
||||
setBlocks([...blocks, newBlock]);
|
||||
};
|
||||
@@ -585,6 +587,17 @@ export default function LessonEditor({ params }: { params: { id: string; lessonI
|
||||
onChange={(updates) => updateBlock(block.id, updates)}
|
||||
/>
|
||||
)}
|
||||
{block.type === 'audio-response' && (
|
||||
<AudioResponseBlock
|
||||
id={block.id}
|
||||
title={block.title}
|
||||
prompt={block.prompt || ""}
|
||||
keywords={block.keywords || []}
|
||||
timeLimit={block.timeLimit}
|
||||
editMode={editMode}
|
||||
onChange={(updates) => updateBlock(block.id, updates)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
@@ -657,6 +670,13 @@ export default function LessonEditor({ params }: { params: { id: string; lessonI
|
||||
<span className="text-2xl group-hover:scale-110 transition-transform">📚</span>
|
||||
<span className="text-[10px] font-bold uppercase tracking-widest text-gray-400">Reading</span>
|
||||
</button>
|
||||
<button
|
||||
onClick={() => addBlock('audio-response')}
|
||||
className="flex flex-col items-center gap-2 p-6 glass hover:border-purple-500/50 transition-all group w-32"
|
||||
>
|
||||
<span className="text-2xl group-hover:scale-110 transition-transform">🎤</span>
|
||||
<span className="text-[10px] font-bold uppercase tracking-widest text-gray-400">Audio</span>
|
||||
</button>
|
||||
|
||||
<div className="w-px h-12 bg-white/5"></div>
|
||||
|
||||
|
||||
@@ -149,7 +149,7 @@ export default function QuizBlock({ id, title, quizData, editMode, onChange }: Q
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
q.options.map((opt, oIdx) => (
|
||||
q.options?.map((opt, oIdx) => (
|
||||
<div key={oIdx} className="flex gap-3 items-center group/opt">
|
||||
<input
|
||||
type={q.type === 'multiple-select' ? "checkbox" : "radio"}
|
||||
@@ -207,38 +207,44 @@ export default function QuizBlock({ id, title, quizData, editMode, onChange }: Q
|
||||
<div key={q.id} className="space-y-4 p-6 glass border-white/5 rounded-2xl">
|
||||
<h4 className="font-bold text-xl text-gray-100 leading-tight">{q.question}</h4>
|
||||
<div className="grid gap-3">
|
||||
{q.options.map((opt, oIdx) => {
|
||||
const isSelected = userAnswers[q.id]?.includes(oIdx);
|
||||
const isCorrect = q.correct?.includes(oIdx);
|
||||
const isActuallyCorrect = isCorrect && isSelected;
|
||||
const isWrongSelection = !isCorrect && isSelected;
|
||||
const missedCorrect = isCorrect && !isSelected;
|
||||
{q.options && q.options.length > 0 ? (
|
||||
q.options.map((opt, oIdx) => {
|
||||
const isSelected = userAnswers[q.id]?.includes(oIdx);
|
||||
const isCorrect = q.correct?.includes(oIdx);
|
||||
const isActuallyCorrect = isCorrect && isSelected;
|
||||
const isWrongSelection = !isCorrect && isSelected;
|
||||
const missedCorrect = isCorrect && !isSelected;
|
||||
|
||||
let style = "glass border-white/10 hover:bg-white/5";
|
||||
if (submitted) {
|
||||
if (isActuallyCorrect) style = "bg-green-500/20 border-green-500 text-green-400";
|
||||
else if (isWrongSelection) style = "bg-red-500/20 border-red-500 text-red-100";
|
||||
else if (missedCorrect) style = "border-orange-500/50 text-orange-400 animate-pulse";
|
||||
else style = "opacity-50 grayscale border-white/5";
|
||||
} else if (isSelected) {
|
||||
style = "bg-blue-500/20 border-blue-500 text-white shadow-[0_0_20px_rgba(59,130,246,0.2)]";
|
||||
}
|
||||
let style = "glass border-white/10 hover:bg-white/5";
|
||||
if (submitted) {
|
||||
if (isActuallyCorrect) style = "bg-green-500/20 border-green-500 text-green-400";
|
||||
else if (isWrongSelection) style = "bg-red-500/20 border-red-500 text-red-100";
|
||||
else if (missedCorrect) style = "border-orange-500/50 text-orange-400 animate-pulse";
|
||||
else style = "opacity-50 grayscale border-white/5";
|
||||
} else if (isSelected) {
|
||||
style = "bg-blue-500/20 border-blue-500 text-white shadow-[0_0_20px_rgba(59,130,246,0.2)]";
|
||||
}
|
||||
|
||||
return (
|
||||
<button
|
||||
key={oIdx}
|
||||
onClick={() => handleAnswer(q.id, oIdx, q.type === 'multiple-select')}
|
||||
className={`p-5 rounded-xl border transition-all text-left text-sm font-bold ${style}`}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span>{opt}</span>
|
||||
{submitted && isActuallyCorrect && <span>✅</span>}
|
||||
{submitted && isWrongSelection && <span>❌</span>}
|
||||
{submitted && missedCorrect && <span className="text-[10px] uppercase font-black tracking-tighter">Correct Answer</span>}
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
return (
|
||||
<button
|
||||
key={oIdx}
|
||||
onClick={() => handleAnswer(q.id, oIdx, q.type === 'multiple-select')}
|
||||
className={`p-5 rounded-xl border transition-all text-left text-sm font-bold ${style}`}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span>{opt}</span>
|
||||
{submitted && isActuallyCorrect && <span>✅</span>}
|
||||
{submitted && isWrongSelection && <span>❌</span>}
|
||||
{submitted && missedCorrect && <span className="text-[10px] uppercase font-black tracking-tighter">Correct Answer</span>}
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
<div className="text-xs text-orange-400/50 font-medium italic p-4 border border-dashed border-orange-500/20 rounded-xl">
|
||||
No options generated for this question.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
@@ -44,7 +44,7 @@ export interface QuizQuestion {
|
||||
|
||||
export interface Block {
|
||||
id: string;
|
||||
type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'document' | 'video_marker';
|
||||
type: 'description' | 'media' | 'quiz' | 'fill-in-the-blanks' | 'matching' | 'ordering' | 'short-answer' | 'document' | 'video_marker' | 'audio-response' | 'memory-match';
|
||||
title?: string;
|
||||
content?: string;
|
||||
url?: string;
|
||||
@@ -57,6 +57,8 @@ export interface Block {
|
||||
items?: string[];
|
||||
prompt?: string;
|
||||
correctAnswers?: string[];
|
||||
keywords?: string[];
|
||||
timeLimit?: number;
|
||||
markers?: {
|
||||
timestamp: number;
|
||||
question: string;
|
||||
|
||||
Reference in New Issue
Block a user