Add docker-compose.local.yml for local development setup
- Introduced a new docker-compose.local.yml file to facilitate local development. - Disabled nginx-proxy and acme-companion services for local use. - Exposed database and application ports directly to the host for easier access. - Configured PostgreSQL to be accessible on localhost:5433. - Mapped application ports for studio and experience services.
This commit is contained in:
@@ -193,11 +193,55 @@ echo ""
|
|||||||
echo "✅ Archivos sincronizados exitosamente!"
|
echo "✅ Archivos sincronizados exitosamente!"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# BUILD LOCAL + STREAM AL REMOTO
|
||||||
|
# ============================================================================
|
||||||
|
if [ "$BUILD_LOCAL" = "true" ]; then
|
||||||
|
echo "========================================"
|
||||||
|
echo " Compilando imágenes localmente"
|
||||||
|
echo "========================================"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Exportar build args desde .env local
|
||||||
|
export NEXT_PUBLIC_CMS_API_URL=$(grep '^NEXT_PUBLIC_CMS_API_URL=' .env | cut -d'=' -f2-)
|
||||||
|
export NEXT_PUBLIC_LMS_API_URL=$(grep '^NEXT_PUBLIC_LMS_API_URL=' .env | cut -d'=' -f2-)
|
||||||
|
export NEXT_PUBLIC_STUDIO_DOMAIN=$(grep '^NEXT_PUBLIC_STUDIO_DOMAIN=' .env | cut -d'=' -f2-)
|
||||||
|
export NEXT_PUBLIC_LEARNING_DOMAIN=$(grep '^NEXT_PUBLIC_LEARNING_DOMAIN=' .env | cut -d'=' -f2-)
|
||||||
|
export COMPOSE_PARALLEL_LIMIT=1
|
||||||
|
|
||||||
|
echo " CMS API URL : $NEXT_PUBLIC_CMS_API_URL"
|
||||||
|
echo " LMS API URL : $NEXT_PUBLIC_LMS_API_URL"
|
||||||
|
echo " Studio : $NEXT_PUBLIC_STUDIO_DOMAIN"
|
||||||
|
echo " Learning : $NEXT_PUBLIC_LEARNING_DOMAIN"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo "🔨 Compilando imagen studio..."
|
||||||
|
docker compose -f docker-compose.yml build --no-cache studio
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "🔨 Compilando imagen experience..."
|
||||||
|
docker compose -f docker-compose.yml build --no-cache experience
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "📤 Transfiriendo studio al servidor (streaming SSH)..."
|
||||||
|
docker save openccb-studio | gzip | \
|
||||||
|
ssh -i "$PEM_PATH" "$REMOTE_USER@$REMOTE_HOST" "gunzip | sudo docker load"
|
||||||
|
|
||||||
|
echo "📤 Transfiriendo experience al servidor (streaming SSH)..."
|
||||||
|
docker save openccb-experience | gzip | \
|
||||||
|
ssh -i "$PEM_PATH" "$REMOTE_USER@$REMOTE_HOST" "gunzip | sudo docker load"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "✅ Imágenes transferidas correctamente"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# SCRIPT REMOTO PARA GESTIÓN DE CONTENEDORES
|
# SCRIPT REMOTO PARA GESTIÓN DE CONTENEDORES
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
echo "🔧 Ejecutando gestión de contenedores en remoto..."
|
echo "🔧 Ejecutando gestión de contenedores en remoto..."
|
||||||
echo ""
|
echo ""
|
||||||
|
echo ""
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# PREGUNTAR DATOS DEL ADMINISTRADOR (LOCAL)
|
# PREGUNTAR DATOS DEL ADMINISTRADOR (LOCAL)
|
||||||
@@ -313,6 +357,25 @@ else
|
|||||||
echo "✅ Configuración: HTTP (sin SSL)"
|
echo "✅ Configuración: HTTP (sin SSL)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "----------------------------------------"
|
||||||
|
echo "Dónde compilar las imágenes Docker"
|
||||||
|
echo "----------------------------------------"
|
||||||
|
echo ""
|
||||||
|
echo "¿Compilar imágenes en esta máquina y enviar al servidor?"
|
||||||
|
echo " - local: Compilar aquí y transferir vía SSH (recomendado - CPU i7 >> t3a.large)"
|
||||||
|
echo " - remote: El servidor compila (más lento en t3a.large 2vCPU/8GB)"
|
||||||
|
echo ""
|
||||||
|
read -p "¿Compilar localmente? [Y/n]: " BUILD_LOCAL_CHOICE
|
||||||
|
BUILD_LOCAL_CHOICE=${BUILD_LOCAL_CHOICE:-Y}
|
||||||
|
if [[ "$BUILD_LOCAL_CHOICE" =~ ^[Yy]$ ]]; then
|
||||||
|
BUILD_LOCAL="true"
|
||||||
|
echo "✅ Compilación local - imágenes se streamearan via SSH"
|
||||||
|
else
|
||||||
|
BUILD_LOCAL="false"
|
||||||
|
echo "✅ El servidor compilará las imágenes"
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "========================================"
|
echo "========================================"
|
||||||
echo " Resumen de Configuración"
|
echo " Resumen de Configuración"
|
||||||
@@ -325,6 +388,7 @@ echo " Protocolo: $PROTOCOL"
|
|||||||
echo " SSL Staging: $LETSENCRYPT_STAGING"
|
echo " SSL Staging: $LETSENCRYPT_STAGING"
|
||||||
echo " Preservar SSL: $PRESERVE_SSL_CERTS"
|
echo " Preservar SSL: $PRESERVE_SSL_CERTS"
|
||||||
echo " Reiniciar DB: $RESET_DATABASE"
|
echo " Reiniciar DB: $RESET_DATABASE"
|
||||||
|
echo " Compilar local: $BUILD_LOCAL"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
# Crear script remoto en un archivo temporal
|
# Crear script remoto en un archivo temporal
|
||||||
@@ -338,6 +402,7 @@ PRESERVE_SSL_CERTS=$PRESERVE_SSL_CERTS
|
|||||||
PROTOCOL=$PROTOCOL
|
PROTOCOL=$PROTOCOL
|
||||||
STUDIO_DOMAIN=$STUDIO_DOMAIN
|
STUDIO_DOMAIN=$STUDIO_DOMAIN
|
||||||
LEARNING_DOMAIN=$LEARNING_DOMAIN
|
LEARNING_DOMAIN=$LEARNING_DOMAIN
|
||||||
|
BUILD_LOCAL=$BUILD_LOCAL
|
||||||
|
|
||||||
cd /var/www/openccb
|
cd /var/www/openccb
|
||||||
|
|
||||||
@@ -663,9 +728,13 @@ $DOCKER_CMD builder prune -f 2>/dev/null || true
|
|||||||
# Evitar builds concurrentes que puedan competir por cachés compartidas
|
# Evitar builds concurrentes que puedan competir por cachés compartidas
|
||||||
export COMPOSE_PARALLEL_LIMIT=1
|
export COMPOSE_PARALLEL_LIMIT=1
|
||||||
|
|
||||||
# Reconstruir con las URLs correctas (sin cache para asegurar que tome los cambios)
|
if [ "$BUILD_LOCAL" = "true" ]; then
|
||||||
echo "Reconstruyendo contenedores con las URLs configuradas..."
|
echo "Imágenes pre-cargadas localmente - saltando build remoto"
|
||||||
run_docker_compose build --no-cache studio experience db
|
else
|
||||||
|
# Reconstruir con las URLs correctas (sin cache para asegurar que tome los cambios)
|
||||||
|
echo "Reconstruyendo contenedores con las URLs configuradas..."
|
||||||
|
run_docker_compose build --no-cache studio experience db
|
||||||
|
fi
|
||||||
|
|
||||||
# Iniciar nginx-proxy y acme-companion primero
|
# Iniciar nginx-proxy y acme-companion primero
|
||||||
echo "Iniciando nginx-proxy y acme-companion - SSL..."
|
echo "Iniciando nginx-proxy y acme-companion - SSL..."
|
||||||
@@ -702,6 +771,17 @@ if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# SIEMPRE sincronizar la contraseña del rol con el .env actual
|
||||||
|
# Evita el error 28P01 (password authentication failed) cuando el deploy regenera DB_PASSWORD
|
||||||
|
CURRENT_DB_PASS=$(grep "^DB_PASSWORD=" .env | cut -d"=" -f2-)
|
||||||
|
if [ -n "$CURRENT_DB_PASS" ]; then
|
||||||
|
echo " Sincronizando contraseña del rol 'user' con .env..."
|
||||||
|
$DOCKER_CMD exec openccb-db psql -U user -d postgres \
|
||||||
|
-c "ALTER USER \"user\" WITH PASSWORD '$CURRENT_DB_PASS';" >/dev/null 2>&1 && \
|
||||||
|
echo " ✅ Contraseña sincronizada" || \
|
||||||
|
echo " ⚠️ No se pudo sincronizar contraseña (el rol usará la del volumen)"
|
||||||
|
fi
|
||||||
|
|
||||||
# Verificar e crear bases de datos
|
# Verificar e crear bases de datos
|
||||||
CMS_EXISTS=$($DOCKER_CMD exec openccb-db psql -U user -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='openccb_cms';" 2>/dev/null | tr -d '[:space:]')
|
CMS_EXISTS=$($DOCKER_CMD exec openccb-db psql -U user -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='openccb_cms';" 2>/dev/null | tr -d '[:space:]')
|
||||||
LMS_EXISTS=$($DOCKER_CMD exec openccb-db psql -U user -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='openccb_lms';" 2>/dev/null | tr -d '[:space:]')
|
LMS_EXISTS=$($DOCKER_CMD exec openccb-db psql -U user -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname='openccb_lms';" 2>/dev/null | tr -d '[:space:]')
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# docker-compose.local.yml — Override para desarrollo local
|
||||||
|
# Uso: docker compose -f docker-compose.yml -f docker-compose.local.yml [comando]
|
||||||
|
#
|
||||||
|
# Diferencias con producción:
|
||||||
|
# - nginx-proxy y acme-companion deshabilitados (profiles: production)
|
||||||
|
# - Puertos expuestos directamente al host
|
||||||
|
# - DB accesible en localhost:5433
|
||||||
|
|
||||||
|
services:
|
||||||
|
nginx-proxy:
|
||||||
|
profiles:
|
||||||
|
- production
|
||||||
|
|
||||||
|
acme-companion:
|
||||||
|
profiles:
|
||||||
|
- production
|
||||||
|
|
||||||
|
db:
|
||||||
|
ports:
|
||||||
|
- "5433:5432"
|
||||||
|
|
||||||
|
studio:
|
||||||
|
ports:
|
||||||
|
- "3000:3000"
|
||||||
|
- "3001:3001"
|
||||||
|
|
||||||
|
experience:
|
||||||
|
ports:
|
||||||
|
- "3003:3003"
|
||||||
|
- "3002:3002"
|
||||||
+23
-20
@@ -1,30 +1,33 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
# OpenCCB Unified Installation Script
|
# OpenCCB Local Development Setup
|
||||||
# This script automates the setup of OpenCCB:
|
# Levanta el stack completo en local usando Docker:
|
||||||
# 1. Prerequisite checks (Rust, Node.js, Docker, sqlx-cli)
|
# - PostgreSQL en localhost:5433
|
||||||
# 2. Hardware detection (NVIDIA GPU vs CPU)
|
# - CMS API en localhost:3001
|
||||||
# 3. Environment configuration (.env) - Dev/Prod support
|
# - Studio (Next.js) en localhost:3000
|
||||||
# 4. Database creation and migrations (CMS, LMS, AI Bridge)
|
# - LMS API en localhost:3002
|
||||||
# 5. System initialization (Admin account and Organization)
|
# - Experience (Next.js) en localhost:3003
|
||||||
# 6. Optional: Production deployment with SSH sync
|
#
|
||||||
# Version: 3.0 - Dev/Prod + Deployment Support
|
# Uso: ./install.sh [--fast] [--clean]
|
||||||
|
# --fast Salta instalación de dependencias del sistema
|
||||||
|
# --clean Elimina volúmenes de DB antes de iniciar (instalación limpia)
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
# CONFIGURACIÓN DE PRODUCCIÓN
|
# PARÁMETROS LOCALES (no editar — se derivan del docker-compose.local.yml)
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
PEM_PATH="ubuntu.pem"
|
LOCAL_DB_PORT="5433"
|
||||||
REMOTE_USER="ubuntu"
|
LOCAL_DB_USER="user"
|
||||||
REMOTE_HOST="ec2-18-224-137-67.us-east-2.compute.amazonaws.com"
|
LOCAL_DB_PASS="password"
|
||||||
REMOTE_PATH="/var/www/openccb"
|
LOCAL_CMS_URL="http://localhost:3001"
|
||||||
# ============================================================================
|
LOCAL_LMS_URL="http://localhost:3002/lms-api"
|
||||||
# CONFIGURACIÓN SAM (Sistema de Administración Académica)
|
LOCAL_STUDIO_DOMAIN="localhost"
|
||||||
# ============================================================================
|
LOCAL_LEARNING_DOMAIN="localhost"
|
||||||
# URL de conexión a la base de datos SAM externa
|
DB_CONTAINER="openccb-db"
|
||||||
# Formato: postgresql://usuario:contraseña@host:puerto/sige_sam_v3
|
COMPOSE_LOCAL="docker compose -f docker-compose.yml -f docker-compose.local.yml"
|
||||||
SAM_DATABASE_URL=""
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
echo "===================================================="
|
echo "===================================================="
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ use std::env;
|
|||||||
use std::path::Path as StdPath;
|
use std::path::Path as StdPath;
|
||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use tokio::io::AsyncWriteExt;
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use tokio::task::JoinSet;
|
||||||
|
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub struct AssetUploadResponse {
|
pub struct AssetUploadResponse {
|
||||||
@@ -161,10 +162,28 @@ async fn maybe_push_local_file_to_s3(
|
|||||||
let client = build_s3_client(&settings).await?;
|
let client = build_s3_client(&settings).await?;
|
||||||
let key = build_s3_object_key(org_id, course_id, storage_filename);
|
let key = build_s3_object_key(org_id, course_id, storage_filename);
|
||||||
|
|
||||||
|
let (storage_path, public_url) = push_bytes_to_s3(
|
||||||
|
&client,
|
||||||
|
&settings,
|
||||||
|
&key,
|
||||||
|
mimetype,
|
||||||
|
bytes,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(Some((storage_path, public_url)))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn push_bytes_to_s3(
|
||||||
|
client: &S3Client,
|
||||||
|
settings: &S3Settings,
|
||||||
|
key: &str,
|
||||||
|
mimetype: &str,
|
||||||
|
bytes: Vec<u8>,
|
||||||
|
) -> Result<(String, String), (StatusCode, String)> {
|
||||||
client
|
client
|
||||||
.put_object()
|
.put_object()
|
||||||
.bucket(&settings.bucket)
|
.bucket(&settings.bucket)
|
||||||
.key(&key)
|
.key(key)
|
||||||
.content_type(mimetype)
|
.content_type(mimetype)
|
||||||
.body(bytes.into())
|
.body(bytes.into())
|
||||||
.send()
|
.send()
|
||||||
@@ -172,8 +191,8 @@ async fn maybe_push_local_file_to_s3(
|
|||||||
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("S3 upload failed: {}", e)))?;
|
.map_err(|e| (StatusCode::BAD_GATEWAY, format!("S3 upload failed: {}", e)))?;
|
||||||
|
|
||||||
let storage_path = format!("s3://{}/{}", settings.bucket, key);
|
let storage_path = format!("s3://{}/{}", settings.bucket, key);
|
||||||
let public_url = build_s3_public_url(&settings, &key);
|
let public_url = build_s3_public_url(settings, key);
|
||||||
Ok(Some((storage_path, public_url)))
|
Ok((storage_path, public_url))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_storage_path(storage_path: &str) -> Result<(), (StatusCode, String)> {
|
async fn delete_storage_path(storage_path: &str) -> Result<(), (StatusCode, String)> {
|
||||||
@@ -637,6 +656,149 @@ struct ZipEntryData {
|
|||||||
is_flv: bool,
|
is_flv: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn process_zip_entry_without_rag(
|
||||||
|
entry: ZipEntryData,
|
||||||
|
org_id: Uuid,
|
||||||
|
user_id: Uuid,
|
||||||
|
pool: PgPool,
|
||||||
|
course_id: Option<Uuid>,
|
||||||
|
english_level: Option<String>,
|
||||||
|
sam_plan_id: Option<i32>,
|
||||||
|
sam_course_id: Option<i32>,
|
||||||
|
split_midpoint: Option<i32>,
|
||||||
|
sam_course_id_r1: Option<i32>,
|
||||||
|
sam_course_id_r2: Option<i32>,
|
||||||
|
s3_settings: Option<S3Settings>,
|
||||||
|
s3_client: Option<S3Client>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let ZipEntryData {
|
||||||
|
entry_name,
|
||||||
|
safe_filename,
|
||||||
|
content,
|
||||||
|
unit_number,
|
||||||
|
guessed_mimetype,
|
||||||
|
is_flv,
|
||||||
|
..
|
||||||
|
} = entry;
|
||||||
|
|
||||||
|
let effective_sam_course_id = match (split_midpoint, unit_number) {
|
||||||
|
(Some(mid), Some(u)) => {
|
||||||
|
if u <= mid { sam_course_id_r1 } else { sam_course_id_r2 }
|
||||||
|
}
|
||||||
|
_ => sam_course_id,
|
||||||
|
};
|
||||||
|
|
||||||
|
let asset_id = Uuid::new_v4();
|
||||||
|
|
||||||
|
let (storage_path, stored_filename, mimetype) = if is_flv {
|
||||||
|
let temp_storage_filename = format!("{}.flv", asset_id);
|
||||||
|
let temp_storage_path = format!("uploads/{}", temp_storage_filename);
|
||||||
|
tokio::fs::write(&temp_storage_path, &content)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{}: local write failed ({})", entry_name, e))?;
|
||||||
|
|
||||||
|
let final_storage_filename = format!("{}.mp4", asset_id);
|
||||||
|
let final_storage_path = format!("uploads/{}", final_storage_filename);
|
||||||
|
if let Err((_, msg)) = transcode_flv_to_mp4(&temp_storage_path, &final_storage_path).await {
|
||||||
|
let _ = tokio::fs::remove_file(&temp_storage_path).await;
|
||||||
|
return Err(format!("{}: flv transcode failed ({})", entry_name, msg));
|
||||||
|
}
|
||||||
|
let _ = tokio::fs::remove_file(&temp_storage_path).await;
|
||||||
|
|
||||||
|
(
|
||||||
|
final_storage_path,
|
||||||
|
replace_extension(&safe_filename, "mp4"),
|
||||||
|
"video/mp4".to_string(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let extension = StdPath::new(&safe_filename)
|
||||||
|
.extension()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.unwrap_or("");
|
||||||
|
|
||||||
|
let storage_filename = if extension.is_empty() {
|
||||||
|
asset_id.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{}.{}", asset_id, extension)
|
||||||
|
};
|
||||||
|
let storage_path = format!("uploads/{}", storage_filename);
|
||||||
|
|
||||||
|
(storage_path, safe_filename.clone(), guessed_mimetype)
|
||||||
|
};
|
||||||
|
|
||||||
|
let storage_filename_for_s3 = StdPath::new(&storage_path)
|
||||||
|
.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let (db_storage_path, persisted_size, _asset_public_url) = if !storage_filename_for_s3.is_empty() {
|
||||||
|
if let (Some(settings), Some(client)) = (s3_settings.as_ref(), s3_client.as_ref()) {
|
||||||
|
let key = build_s3_object_key(org_id, course_id, &storage_filename_for_s3);
|
||||||
|
let upload_bytes = if is_flv {
|
||||||
|
tokio::fs::read(&storage_path)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{}: local read failed ({})", entry_name, e))?
|
||||||
|
} else {
|
||||||
|
content
|
||||||
|
};
|
||||||
|
let uploaded_len = upload_bytes.len() as i64;
|
||||||
|
let (s3_path, public_url) = push_bytes_to_s3(client, settings, &key, &mimetype, upload_bytes)
|
||||||
|
.await
|
||||||
|
.map_err(|(_, msg)| format!("{}: s3 upload failed ({})", entry_name, msg))?;
|
||||||
|
|
||||||
|
if is_flv {
|
||||||
|
let _ = tokio::fs::remove_file(&storage_path).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
(s3_path, uploaded_len, public_url)
|
||||||
|
} else {
|
||||||
|
if !is_flv {
|
||||||
|
tokio::fs::write(&storage_path, &content)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{}: local write failed ({})", entry_name, e))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let size = tokio::fs::metadata(&storage_path)
|
||||||
|
.await
|
||||||
|
.map(|m| m.len() as i64)
|
||||||
|
.unwrap_or(content.len() as i64);
|
||||||
|
|
||||||
|
(
|
||||||
|
storage_path.clone(),
|
||||||
|
size,
|
||||||
|
format!("/assets/{}", storage_filename_for_s3),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
(storage_path.clone(), content.len() as i64, storage_path.clone())
|
||||||
|
};
|
||||||
|
|
||||||
|
sqlx::query(
|
||||||
|
r#"
|
||||||
|
INSERT INTO assets (id, organization_id, uploaded_by, course_id, english_level, sam_plan_id, sam_course_id, unit_number, filename, storage_path, mimetype, size_bytes)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.bind(asset_id)
|
||||||
|
.bind(org_id)
|
||||||
|
.bind(user_id)
|
||||||
|
.bind(course_id)
|
||||||
|
.bind(&english_level)
|
||||||
|
.bind(sam_plan_id)
|
||||||
|
.bind(effective_sam_course_id)
|
||||||
|
.bind(unit_number)
|
||||||
|
.bind(&stored_filename)
|
||||||
|
.bind(&db_storage_path)
|
||||||
|
.bind(&mimetype)
|
||||||
|
.bind(persisted_size)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{}: db insert failed ({})", entry_name, e))?;
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn import_assets_zip(
|
pub async fn import_assets_zip(
|
||||||
Org(org_ctx): Org,
|
Org(org_ctx): Org,
|
||||||
claims: Claims,
|
claims: Claims,
|
||||||
@@ -824,6 +986,17 @@ pub async fn import_assets_zip(
|
|||||||
// Sort: audio/video first so their asset IDs are known when text is ingested
|
// Sort: audio/video first so their asset IDs are known when text is ingested
|
||||||
all_entries.sort_by_key(|e| if e.is_audio_video { 0usize } else { 1 });
|
all_entries.sort_by_key(|e| if e.is_audio_video { 0usize } else { 1 });
|
||||||
|
|
||||||
|
let s3_settings = get_s3_settings();
|
||||||
|
let s3_client = if let Some(settings) = &s3_settings {
|
||||||
|
Some(build_s3_client(settings).await?)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
tokio::fs::create_dir_all("uploads")
|
||||||
|
.await
|
||||||
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
|
|
||||||
// ── Phase 2: process entries ───────────────────────────────────────────────
|
// ── Phase 2: process entries ───────────────────────────────────────────────
|
||||||
let mut imported_assets = 0usize;
|
let mut imported_assets = 0usize;
|
||||||
let mut rag_ingested_assets = 0usize;
|
let mut rag_ingested_assets = 0usize;
|
||||||
@@ -847,6 +1020,70 @@ pub async fn import_assets_zip(
|
|||||||
let ollama_url = ai::get_ollama_url();
|
let ollama_url = ai::get_ollama_url();
|
||||||
let model = ai::get_embedding_model();
|
let model = ai::get_embedding_model();
|
||||||
|
|
||||||
|
if !ingest_rag {
|
||||||
|
let org_id = org_ctx.id;
|
||||||
|
let user_id = claims.sub;
|
||||||
|
let concurrency = env::var("ZIP_IMPORT_CONCURRENCY")
|
||||||
|
.ok()
|
||||||
|
.and_then(|v| v.parse::<usize>().ok())
|
||||||
|
.map(|v| v.clamp(1, 16))
|
||||||
|
.unwrap_or(4);
|
||||||
|
|
||||||
|
let mut join_set: JoinSet<Result<(), String>> = JoinSet::new();
|
||||||
|
|
||||||
|
for entry in all_entries {
|
||||||
|
while join_set.len() >= concurrency {
|
||||||
|
match join_set.join_next().await {
|
||||||
|
Some(Ok(Ok(()))) => imported_assets += 1,
|
||||||
|
Some(Ok(Err(msg))) => failed_entries.push(msg),
|
||||||
|
Some(Err(e)) => failed_entries.push(format!("zip worker failed: {}", e)),
|
||||||
|
None => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let pool_cl = pool.clone();
|
||||||
|
let english_level_cl = english_level.clone();
|
||||||
|
let s3_settings_cl = s3_settings.clone();
|
||||||
|
let s3_client_cl = s3_client.clone();
|
||||||
|
|
||||||
|
join_set.spawn(async move {
|
||||||
|
process_zip_entry_without_rag(
|
||||||
|
entry,
|
||||||
|
org_id,
|
||||||
|
user_id,
|
||||||
|
pool_cl,
|
||||||
|
course_id,
|
||||||
|
english_level_cl,
|
||||||
|
sam_plan_id,
|
||||||
|
sam_course_id,
|
||||||
|
split_midpoint,
|
||||||
|
sam_course_id_r1,
|
||||||
|
sam_course_id_r2,
|
||||||
|
s3_settings_cl,
|
||||||
|
s3_client_cl,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
while let Some(result) = join_set.join_next().await {
|
||||||
|
match result {
|
||||||
|
Ok(Ok(())) => imported_assets += 1,
|
||||||
|
Ok(Err(msg)) => failed_entries.push(msg),
|
||||||
|
Err(e) => failed_entries.push(format!("zip worker failed: {}", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = tokio::fs::remove_file(&zip_path).await;
|
||||||
|
|
||||||
|
return Ok(Json(AssetZipImportResponse {
|
||||||
|
imported_assets,
|
||||||
|
rag_ingested_assets: 0,
|
||||||
|
rag_chunks_ingested: 0,
|
||||||
|
failed_entries,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
for entry in all_entries {
|
for entry in all_entries {
|
||||||
let ZipEntryData {
|
let ZipEntryData {
|
||||||
entry_name,
|
entry_name,
|
||||||
@@ -871,9 +1108,6 @@ pub async fn import_assets_zip(
|
|||||||
let (storage_path, stored_filename, mimetype) = if is_flv {
|
let (storage_path, stored_filename, mimetype) = if is_flv {
|
||||||
let temp_storage_filename = format!("{}.flv", asset_id);
|
let temp_storage_filename = format!("{}.flv", asset_id);
|
||||||
let temp_storage_path = format!("uploads/{}", temp_storage_filename);
|
let temp_storage_path = format!("uploads/{}", temp_storage_filename);
|
||||||
tokio::fs::create_dir_all("uploads")
|
|
||||||
.await
|
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
||||||
tokio::fs::write(&temp_storage_path, &content)
|
tokio::fs::write(&temp_storage_path, &content)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||||
@@ -901,13 +1135,6 @@ pub async fn import_assets_zip(
|
|||||||
};
|
};
|
||||||
let storage_path = format!("uploads/{}", storage_filename);
|
let storage_path = format!("uploads/{}", storage_filename);
|
||||||
|
|
||||||
tokio::fs::create_dir_all("uploads")
|
|
||||||
.await
|
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
||||||
tokio::fs::write(&storage_path, &content)
|
|
||||||
.await
|
|
||||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
||||||
|
|
||||||
(storage_path, safe_filename.clone(), guessed_mimetype)
|
(storage_path, safe_filename.clone(), guessed_mimetype)
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -918,18 +1145,44 @@ pub async fn import_assets_zip(
|
|||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let (db_storage_path, asset_public_url) = if !storage_filename_for_s3.is_empty() {
|
let (db_storage_path, asset_public_url) = if !storage_filename_for_s3.is_empty() {
|
||||||
if let Some((s3_path, public_url)) = maybe_push_local_file_to_s3(
|
if let (Some(settings), Some(client)) = (s3_settings.as_ref(), s3_client.as_ref()) {
|
||||||
&storage_path,
|
let key = build_s3_object_key(org_ctx.id, course_id, &storage_filename_for_s3);
|
||||||
&storage_filename_for_s3,
|
let upload_bytes = if is_flv {
|
||||||
&mimetype,
|
match tokio::fs::read(&storage_path).await {
|
||||||
org_ctx.id,
|
Ok(bytes) => bytes,
|
||||||
course_id,
|
Err(e) => {
|
||||||
)
|
failed_entries.push(format!("{}: local read failed ({})", entry_name, e));
|
||||||
.await?
|
let _ = tokio::fs::remove_file(&storage_path).await;
|
||||||
{
|
continue;
|
||||||
let _ = tokio::fs::remove_file(&storage_path).await;
|
}
|
||||||
(s3_path, public_url)
|
}
|
||||||
|
} else {
|
||||||
|
content.clone()
|
||||||
|
};
|
||||||
|
|
||||||
|
match push_bytes_to_s3(client, settings, &key, &mimetype, upload_bytes).await {
|
||||||
|
Ok((s3_path, public_url)) => {
|
||||||
|
if is_flv {
|
||||||
|
let _ = tokio::fs::remove_file(&storage_path).await;
|
||||||
|
}
|
||||||
|
(s3_path, public_url)
|
||||||
|
}
|
||||||
|
Err((_, msg)) => {
|
||||||
|
if is_flv {
|
||||||
|
let _ = tokio::fs::remove_file(&storage_path).await;
|
||||||
|
}
|
||||||
|
failed_entries.push(format!("{}: s3 upload failed ({})", entry_name, msg));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
|
if !is_flv {
|
||||||
|
if let Err(e) = tokio::fs::write(&storage_path, &content).await {
|
||||||
|
failed_entries.push(format!("{}: local write failed ({})", entry_name, e));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
(
|
(
|
||||||
storage_path.clone(),
|
storage_path.clone(),
|
||||||
format!("/assets/{}", storage_filename_for_s3),
|
format!("/assets/{}", storage_filename_for_s3),
|
||||||
|
|||||||
+60
-45
@@ -980,55 +980,70 @@ export const cmsApi = {
|
|||||||
samCourseIdR2?: number,
|
samCourseIdR2?: number,
|
||||||
): Promise<AssetZipImportResult> => {
|
): Promise<AssetZipImportResult> => {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
const formData = new FormData();
|
const maxNetworkRetries = 2;
|
||||||
formData.append('file', file);
|
|
||||||
formData.append('ingest_rag', ingestRag ? 'true' : 'false');
|
|
||||||
if (courseId) formData.append('course_id', courseId);
|
|
||||||
if (englishLevel) formData.append('english_level', englishLevel);
|
|
||||||
if (samPlanId) formData.append('sam_plan_id', String(samPlanId));
|
|
||||||
if (samCourseId) formData.append('sam_course_id', String(samCourseId));
|
|
||||||
if (splitToRegular) {
|
|
||||||
formData.append('split_to_regular', 'true');
|
|
||||||
if (samCourseIdR1) formData.append('sam_course_id_r1', String(samCourseIdR1));
|
|
||||||
if (samCourseIdR2) formData.append('sam_course_id_r2', String(samCourseIdR2));
|
|
||||||
}
|
|
||||||
|
|
||||||
const xhr = new XMLHttpRequest();
|
const startAttempt = (attempt: number) => {
|
||||||
xhr.open('POST', `${API_BASE_URL}/api/assets/import-zip`);
|
const formData = new FormData();
|
||||||
|
formData.append('file', file);
|
||||||
const token = getToken();
|
formData.append('ingest_rag', ingestRag ? 'true' : 'false');
|
||||||
if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);
|
if (courseId) formData.append('course_id', courseId);
|
||||||
const selectedOrgId = getSelectedOrgId();
|
if (englishLevel) formData.append('english_level', englishLevel);
|
||||||
if (selectedOrgId) xhr.setRequestHeader('X-Organization-Id', selectedOrgId);
|
if (samPlanId) formData.append('sam_plan_id', String(samPlanId));
|
||||||
|
if (samCourseId) formData.append('sam_course_id', String(samCourseId));
|
||||||
xhr.onload = () => {
|
if (splitToRegular) {
|
||||||
if (xhr.status >= 200 && xhr.status < 300) {
|
formData.append('split_to_regular', 'true');
|
||||||
resolve(JSON.parse(xhr.responseText));
|
if (samCourseIdR1) formData.append('sam_course_id_r1', String(samCourseIdR1));
|
||||||
} else {
|
if (samCourseIdR2) formData.append('sam_course_id_r2', String(samCourseIdR2));
|
||||||
let msg = `ZIP import failed (HTTP ${xhr.status})`;
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(xhr.responseText);
|
|
||||||
msg = parsed.message || parsed.error || msg;
|
|
||||||
} catch {
|
|
||||||
const raw = (xhr.responseText || '').trim();
|
|
||||||
if (raw) {
|
|
||||||
const compact = raw.replace(/\s+/g, ' ').slice(0, 240);
|
|
||||||
msg = `${msg}: ${compact}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
reject(new Error(msg));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const xhr = new XMLHttpRequest();
|
||||||
|
xhr.open('POST', `${API_BASE_URL}/api/assets/import-zip`);
|
||||||
|
|
||||||
|
const token = getToken();
|
||||||
|
if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);
|
||||||
|
const selectedOrgId = getSelectedOrgId();
|
||||||
|
if (selectedOrgId) xhr.setRequestHeader('X-Organization-Id', selectedOrgId);
|
||||||
|
|
||||||
|
xhr.onload = () => {
|
||||||
|
if (xhr.status >= 200 && xhr.status < 300) {
|
||||||
|
resolve(JSON.parse(xhr.responseText));
|
||||||
|
} else {
|
||||||
|
let msg = `ZIP import failed (HTTP ${xhr.status})`;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(xhr.responseText);
|
||||||
|
msg = parsed.message || parsed.error || msg;
|
||||||
|
} catch {
|
||||||
|
const raw = (xhr.responseText || '').trim();
|
||||||
|
if (raw) {
|
||||||
|
const compact = raw.replace(/\s+/g, ' ').slice(0, 240);
|
||||||
|
msg = `${msg}: ${compact}`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reject(new Error(msg));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
xhr.onerror = () => {
|
||||||
|
if (attempt < maxNetworkRetries) {
|
||||||
|
const delayMs = 1200 * (attempt + 1);
|
||||||
|
setTimeout(() => startAttempt(attempt + 1), delayMs);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
reject(new Error('Network error'));
|
||||||
|
};
|
||||||
|
|
||||||
|
if (onProgress) {
|
||||||
|
xhr.upload.onprogress = (event) => {
|
||||||
|
if (!event.lengthComputable) return;
|
||||||
|
const pct = Math.round((event.loaded / event.total) * 100);
|
||||||
|
onProgress(Math.max(0, Math.min(100, pct)));
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
xhr.send(formData);
|
||||||
};
|
};
|
||||||
|
|
||||||
xhr.onerror = () => reject(new Error('Network error'));
|
startAttempt(0);
|
||||||
if (onProgress) {
|
|
||||||
xhr.upload.onprogress = (event) => {
|
|
||||||
if (!event.lengthComputable) return;
|
|
||||||
const pct = Math.round((event.loaded / event.total) * 100);
|
|
||||||
onProgress(Math.max(0, Math.min(100, pct)));
|
|
||||||
};
|
|
||||||
}
|
|
||||||
xhr.send(formData);
|
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
uploadAsset: (
|
uploadAsset: (
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user