From 9c72b00b1b05a99abddc11a73c0a3d43ad2bf381 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 12 Feb 2026 13:13:36 -0500 Subject: [PATCH] feat: Implement image fetching and storage system - Add image-fetcher module for downloading and saving images from various sources. - Create storage module for managing image files, including downloading, verifying integrity, and cleaning up orphaned files. - Develop gallery HTML page for displaying images with sorting and filtering options. - Set up RESTful API routes for image management, including fetching, adding tags, and deleting images. - Introduce setup script for initializing the database and configuring image sources. - Implement a batch process for verifying image integrity and cleaning up old images. - Add setup batch script for easy installation and configuration of the image storage system. --- .dockerignore | 4 + DOCKER_DEPLOYMENT.md | 412 +++++++++++++++++++++++++++++ Dockerfile | 9 +- IMAGE_STORAGE_GUIDE.md | 435 ++++++++++++++++++++++++++++++ README.md | 190 +++++++++---- docker-compose.yml | 14 +- image-sources.json | 19 ++ lib/database.js | 342 ++++++++++++++++++++++++ lib/image-fetcher.js | 162 ++++++++++++ lib/storage.js | 244 +++++++++++++++++ package.json | 5 +- public/gallery.html | 587 +++++++++++++++++++++++++++++++++++++++++ public/index.html | 21 ++ public/styles.css | 49 ++++ routes/images.js | 235 +++++++++++++++++ scripts/deploy.ps1 | 16 +- server.js | 87 +++++- setup.bat | 45 ++++ setup.js | 199 ++++++++++++++ 19 files changed, 3004 insertions(+), 71 deletions(-) create mode 100644 DOCKER_DEPLOYMENT.md create mode 100644 IMAGE_STORAGE_GUIDE.md create mode 100644 image-sources.json create mode 100644 lib/database.js create mode 100644 lib/image-fetcher.js create mode 100644 lib/storage.js create mode 100644 public/gallery.html create mode 100644 routes/images.js create mode 100644 setup.bat create mode 100644 setup.js diff --git a/.dockerignore b/.dockerignore index 6daecd0..e6afd53 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,8 @@ node_modules/ *.log README.md +IMAGE_STORAGE_GUIDE.md .gitignore +data/ +setup.bat +scripts/deploy.* diff --git a/DOCKER_DEPLOYMENT.md b/DOCKER_DEPLOYMENT.md new file mode 100644 index 0000000..ff61817 --- /dev/null +++ b/DOCKER_DEPLOYMENT.md @@ -0,0 +1,412 @@ +# ๐Ÿณ Docker Deployment Guide for Image Storage + +This guide covers deployment of the HomeBase image storage system in Docker on your home lab. + +## Prerequisites + +- Docker and Docker Compose installed on your home lab server (beepc) +- SSH access configured (required for deploy script) +- PowerShell 5+ or PowerShell Core for running deploy scripts +- Network access from your dev machine to the home lab + +## Quick Deployment + +### 1. Deploy to Home Lab + +From your local machine, run: + +```powershell +.\scripts\deploy.ps1 +``` + +This script will: +- Copy all project files to your home lab server (spencer@beepc) +- Build the Docker image +- Create and start the container with persistent storage +- Set up systemd service for auto-start + +### 2. After Deployment + +The container will automatically: +- Create the SQLite database +- Create image storage directories +- Start the image fetcher (if configured) +- Be available at `http://homebase.sketchferret.com` + +## Docker Setup Details + +### Persistent Storage + +Images are stored in a Docker volume called `homebase-data`: + +```yaml +volumes: + - homebase-data:/app/data +``` + +This ensures: +- Database persists across container restarts +- Image files are not lost on container recreation +- Data survives container updates + +### Viewing Container Logs + +```bash +# SSH into home lab +ssh spencer@beepc + +# View container logs +docker compose -f ~/homebase/docker-compose.yml logs -f + +# Or use systemd +journalctl -u homebase -f +``` + +### Container Management + +```bash +# Stop container +docker compose -f ~/homebase/docker-compose.yml down + +# Rebuild and restart +docker compose -f ~/homebase/docker-compose.yml up -d --build + +# Check status +docker compose -f ~/homebase/docker-compose.yml ps + +# View stored images +docker exec homebase ls -lh /app/data/images/ +``` + +## Configuration in Docker + +### Image Sources (image-sources.json) + +After deployment, edit the configuration on the remote server: + +```bash +ssh spencer@beepc +nano ~/homebase/image-sources.json +``` + +Then restart the container: + +```bash +cd ~/homebase +docker compose restart app +``` + +### Environment Variables + +Edit `docker-compose.yml` to customize: + +```yaml +environment: + - NODE_ENV=production + - PORT=3001 + - DOMAIN=homebase.sketchferret.com +``` + +## Managing Data + +### Backup Database + +The SQLite database is stored in the Docker volume at `/app/data/homebase.db` + +```bash +# SSH into server +ssh spencer@beepc + +# Backup database +docker run --rm -v homebase-data:/data \ + -v ~/backups:/backup \ + alpine cp /data/homebase.db /backup/homebase.db.$(date +%Y%m%d) + +# Backup images +docker run --rm -v homebase-data:/data \ + -v ~/backups:/backup \ + alpine tar czf /backup/images.$(date +%Y%m%d).tar.gz /data/images/ +``` + +### Restore Database + +```bash +docker run --rm -v homebase-data:/data \ + -v ~/backups:/backup \ + alpine cp /backup/homebase.db.20260211 /data/homebase.db + +# Restart container +docker restart homebase +``` + +### Clean Up Old Images + +```bash +# SSH into server +ssh spencer@beepc + +# Connect to running container +docker exec -it homebase node -e " +const db = require('./lib/database'); +db.initializeDatabase().then(async () => { + const deleted = await db.cleanupOldImages(30); + console.log('Deleted', deleted, 'old images'); + process.exit(0); +}).catch(err => { + console.error(err); + process.exit(1); +}); +" +``` + +## Troubleshooting + +### Container won't start + +```bash +ssh spencer@beepc +cd ~/homebase + +# Check logs +docker compose logs app + +# Rebuild +docker compose up -d --build + +# Check again +docker compose ps +``` + +### Permission errors + +The container runs as the `node` user. Check logs: + +```bash +docker compose exec app ls -la /app/data/ +``` + +All files should be owned by `node`. + +### Can't access via domain + +Check that the service is running: + +```bash +ssh spencer@beepc + +# Check systemd +systemctl status homebase + +# Check Docker +docker ps | grep homebase + +# Test port +curl -i http://localhost:3001/health +``` + +### Database is locked + +This usually means two processes are accessing the database simultaneously: + +```bash +# Check if multiple containers are running +docker ps -a | grep homebase + +# Stop all containers +docker compose down + +# Verify only one is running +docker compose up -d +``` + +## Updating the Deployment + +### Update Code + +After making changes locally: + +```powershell +cd c:\Users\srene\Sources\HomeBase +git commit -am "Add new features" +git push + +# Deploy to home lab +.\scripts\deploy.ps1 +``` + +### Update Dependencies + +If you update `package.json`: + +```powershell +# From your local machine +.\scripts\deploy.ps1 + +# The deploy script will: +# 1. Copy new package.json +# 2. Rebuild the Docker image (npm install runs here) +# 3. Restart the container +``` + +### Manual Updates + +SSH into the server and run: + +```bash +cd ~/homebase + +# Pull latest code +git pull # If using git + +# Or manually copy files via SCP +# Then rebuild: +docker compose up -d --build +``` + +## Systemd Service + +The deployment script sets up a systemd service: + +```bash +# Check service +systemctl status homebase + +# Start service +sudo systemctl start homebase + +# Stop service +sudo systemctl stop homebase + +# View logs +journalctl -u homebase -f +``` + +Service file location: `/etc/systemd/system/homebase.service` + +## Volume Management + +### List volumes + +```bash +docker volume ls | grep homebase +``` + +### Inspect volume + +```bash +docker volume inspect homebase-data +``` + +Shows the mount point on the host server. + +### Manual mount (for recovery) + +```bash +docker run -it --rm -v homebase-data:/data alpine sh +``` + +Then you can inspect files at `/data/` + +## Performance Optimization + +### Monitor Storage Usage + +```bash +curl http://homebase.sketchferret.com/api/stats +``` + +### Cleanup Recommendations + +- **Weekly**: Run cleanup for images older than 7 days +- **Monthly**: Run verification to check corruption +- **Quarterly**: Full database optimization + +```bash +# SSH into server +ssh spencer@beepc +docker exec homebase node -e " +const db = require('./lib/database'); +const storage = require('./lib/storage'); + +db.initializeDatabase().then(async () => { + console.log('Running cleanup...'); + const deleted = await db.cleanupOldImages(7); + const orphaned = await storage.cleanupOrphanedFiles(db.db); + console.log('Deleted:', deleted, 'old images'); + console.log('Removed:', orphaned.cleaned, 'orphaned files'); + process.exit(0); +}).catch(err => { + console.error('Error:', err); + process.exit(1); +}); +" +``` + +## Network & Firewall + +### Open Ports + +Ensure port 3001 is accessible on your home lab network: + +```bash +# Check if port is listening +ssh spencer@beepc +netstat -tlnp | grep 3001 + +# Or use Docker +docker port homebase +``` + +### Domain Configuration + +The app is configured to run on `homebase.sketchferret.com`. Ensure: +- DNS resolves to your home lab IP +- Firewall allows traffic on port 3001 +- Your home lab router has port forwarding (if external access needed) + +## Debugging + +### Enable detailed logging + +```bash +# SSH into server +ssh spencer@beepc + +# View container logs with timestamps +docker compose logs -f --timestamps app + +# Or view the last 100 lines +docker compose logs --tail=100 app +``` + +### Access container shell + +```bash +docker exec -it homebase sh +cd /app +node setup.js config +``` + +### Database inspection + +```bash +# Access SQLite directly +docker exec -it homebase sqlite3 /app/data/homebase.db + +# In sqlite prompt: +sqlite> SELECT COUNT(*) FROM images; +sqlite> SELECT * FROM tags; +sqlite> .quit +``` + +## Best Practices + +1. **Regular Backups**: Back up the database volume weekly +2. **Log Monitoring**: Check logs regularly for errors +3. **Storage Limits**: Monitor storage and clean up old images +4. **Configuration**: Keep `image-sources.json` updated with working URLs +5. **Testing**: Test image fetching after configuration changes + +--- + +For API usage, see [IMAGE_STORAGE_GUIDE.md](IMAGE_STORAGE_GUIDE.md) diff --git a/Dockerfile b/Dockerfile index 480867b..beb09e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,13 @@ RUN npm install --omit=dev COPY . . -# Ensure proper permissions for node user -RUN chown -R node:node /app +# Create data directory for persistent storage +RUN mkdir -p /app/data/images && \ + chown -R node:node /app && \ + chmod -R 755 /app/data + +# Install avahi for mDNS support (.local hostname resolution) +RUN apk add --no-cache avahi avahi-tools libc6-compat EXPOSE 3001 diff --git a/IMAGE_STORAGE_GUIDE.md b/IMAGE_STORAGE_GUIDE.md new file mode 100644 index 0000000..5e40c57 --- /dev/null +++ b/IMAGE_STORAGE_GUIDE.md @@ -0,0 +1,435 @@ +# ๐Ÿ“ธ Image Storage System Guide + +This guide covers the long-term image storage solution integrated into HomeBase. It provides reliable, organized storage for images pulled from web services with built-in corruption detection and ML-friendly tagging. + +## Features + +- **Automatic Image Fetching**: Pull images from web services every 2-3 minutes +- **Corruption Detection**: SHA256 checksums verify data integrity +- **Tag-Based Organization**: Tag images for machine learning model training +- **Efficient Storage**: File-based storage with SQLite metadata database +- **RESTful API**: Complete API for image management and queries +- **Scalable**: Designed to handle thousands of images + +## Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Image Fetcher Service โ”‚ +โ”‚ (Runs every 2-3 minutes, pull from web service) โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”œโ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ โ”‚ โ”‚ โ”‚ + โ–ผ โ–ผ โ–ผ โ–ผ + Download Hash Store File Insert Metadata + Image (SHA256) (data/images) (SQLite) + โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ REST API Endpoints โ”‚ + โ”‚ - List Images โ”‚ + โ”‚ - Add Tags โ”‚ + โ”‚ - Search/Filter โ”‚ + โ”‚ - Verify Integrity โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Quick Start + +### 1. Install Dependencies + +```bash +npm install +``` + +### 2. Initialize the System + +```bash +node setup.js init +``` + +This will: +- Create the SQLite database +- Set up data directory structure +- Load configuration +- Test the system + +### 3. Configure Image Sources + +Edit `image-sources.json`: + +```json +{ + "sources": [ + { + "name": "Webcam Feed", + "url": "https://your-service.com/image.jpg", + "tags": ["webcam", "monitoring"], + "enabled": true + } + ], + "fetchInterval": 0.033 +} +``` + +**Fetch Intervals** (edit `fetchInterval` to customize): +- `0.0167` = 1 second +- `0.033` = 2 seconds (recommended for fast updates) +- `0.05` = 3 seconds +- `2.5` = 2.5 minutes (original default) + +### 4. Start the Server + +```bash +npm start +``` + +The system will automatically start fetching images at the configured interval. + +## Configuration + +### image-sources.json + +```json +{ + "sources": [ + { + "name": "Example Source", + "url": "https://example.com/image.jpg", + "tags": ["tag1", "tag2"], + "enabled": true + } + ], + "fetchInterval": 0.033 +} +``` + +**Options:** +- `fetchInterval`: **Minutes** between fetch cycles. Use decimals for sub-minute intervals: + - `0.0167` = 1 second + - `0.033` = 2 seconds + - `0.05` = 3 seconds + - `0.167` = 10 seconds + - `1` = 1 minute + - `2.5` = 2.5 minutes (original default) + +## API Endpoints + +### List Images + +```bash +GET /api/images?page=1&pageSize=50&sort=fetched_at&order=DESC +GET /api/images?tag=webcam # Filter by tag +GET /api/images?sourceUrl=https://... # Filter by source +``` + +**Response:** +```json +{ + "success": true, + "images": [ + { + "id": 1, + "filename": "image_1234567890_abc123.jpg", + "source_url": "https://...", + "filesize": 102400, + "file_hash": "sha256hash", + "fetched_at": "2023-01-01T12:00:00Z", + "tags": ["webcam", "monitoring"] + } + ], + "pagination": { + "page": 1, + "pageSize": 50, + "total": 240, + "pages": 5 + } +} +``` + +### Get Image Details + +```bash +GET /api/images/{id} +``` + +### Download Image + +```bash +GET /api/images/{id}/download +``` + +### Fetch New Image + +```bash +POST /api/images +Content-Type: application/json + +{ + "source_url": "https://example.com/image.jpg", + "tags": ["tag1", "tag2"] +} +``` + +### Add Tags to Image + +```bash +POST /api/images/{id}/tags +Content-Type: application/json + +{ + "tags": ["newtag1", "newtag2"] +} +``` + +### List All Tags + +```bash +GET /api/tags +``` + +**Response:** +```json +{ + "success": true, + "tags": ["webcam", "monitoring", "test", "example"] +} +``` + +### Storage Statistics + +```bash +GET /api/stats +``` + +**Response:** +```json +{ + "success": true, + "stats": { + "imageCount": 240, + "totalSize": 24576000, + "totalSizeGB": "0.02", + "fileCount": 240 + } +} +``` + +### Verify Image Integrity + +```bash +POST /api/verify +``` + +This checks all images for corruption using their stored checksums. + +### Cleanup Old Images + +```bash +POST /api/cleanup +Content-Type: application/json + +{ + "daysOld": 30 +} +``` + +### Delete Image + +```bash +DELETE /api/images/{id} +``` + +## Corruption Detection + +The system uses SHA256 checksums to detect corruption: + +1. **Storage**: When an image is saved, its SHA256 hash is calculated and stored +2. **Verification**: The `/api/verify` endpoint re-hashes all files and compares with stored hashes +3. **Marking**: Corrupted images are marked in the database and excluded from queries +4. **Recovery**: Corrupted files can be re-fetched using the source URL + +### Manual Verification + +```bash +curl -X POST http://localhost:3001/api/verify +``` + +## Tagging for ML Training + +Tags are essential for organizing training datasets: + +```bash +# Add images with training tags +POST /api/images +{ + "source_url": "https://...", + "tags": ["dataset_v1", "labeled", "weather-sunny"] +} + +# Query all images with specific tag +GET /api/images?tag=weather-sunny + +# Get tag statistics +GET /api/tags +``` + +## Database Schema + +### Images Table + +| Column | Type | Description | +|--------|------|-------------| +| id | INTEGER | Primary key | +| filename | TEXT | Unique filename | +| source_url | TEXT | Original image URL | +| file_path | TEXT | Local file path | +| filesize | INTEGER | File size in bytes | +| file_hash | TEXT | SHA256 hash | +| mime_type | TEXT | Content type | +| fetched_at | DATETIME | When image was fetched | +| is_corrupted | BOOLEAN | Corruption flag | + +### Tags Table + +| Column | Type | Description | +|--------|------|-------------| +| id | INTEGER | Primary key | +| image_id | INTEGER | Foreign key to images | +| tag | TEXT | Tag text | +| created_at | DATETIME | When tag was added | + +## File Structure + +``` +homebase/ +โ”œโ”€โ”€ server.js # Main server +โ”œโ”€โ”€ package.json # Dependencies +โ”œโ”€โ”€ setup.js # Setup script +โ”œโ”€โ”€ image-sources.json # Configuration +โ”œโ”€โ”€ lib/ +โ”‚ โ”œโ”€โ”€ database.js # SQLite operations +โ”‚ โ”œโ”€โ”€ storage.js # File storage operations +โ”‚ โ””โ”€โ”€ image-fetcher.js # Image fetching service +โ”œโ”€โ”€ routes/ +โ”‚ โ””โ”€โ”€ images.js # API routes +โ””โ”€โ”€ data/ # Created at runtime + โ”œโ”€โ”€ homebase.db # SQLite database + โ””โ”€โ”€ images/ # Stored image files +``` + +## Maintenance + +### Manual Setup + +```bash +# Initialize system +node setup.js init + +# Test fetch +node setup.js test https://example.com/image.jpg tag1,tag2 + +# View configuration +node setup.js config +``` + +### Regular Tasks + +```bash +# Daily: Verify integrity +curl -X POST http://localhost:3001/api/verify + +# Weekly: Cleanup old images (30+ days) +curl -X POST http://localhost:3001/api/cleanup -H "Content-Type: application/json" -d '{"daysOld": 30}' + +# Monitor storage +curl http://localhost:3001/api/stats +``` + +## Performance Tips + +1. **Pagination**: Use `pageSize=50` or less for large datasets +2. **Tagging**: Use consistent tag names for easier filtering +3. **Cleanup**: Run cleanup weekly to manage storage +4. **Verification**: Run verification monthly to detect issues early + +## Troubleshooting + +### No images fetching? + +1. Check `image-sources.json` - ensure `enabled: true` +2. Verify URL is accessible +3. Check server logs for errors +4. Test fetch: `node setup.js test https://url.jpg` + +### High storage usage? + +```bash +# Check statistics +curl http://localhost:3001/api/stats + +# Cleanup images older than 7 days +curl -X POST http://localhost:3001/api/cleanup \ + -H "Content-Type: application/json" \ + -d '{"daysOld": 7}' +``` + +### Corrupted files detected? + +```bash +# Verify all images +curl -X POST http://localhost:3001/api/verify + +# Get list of corrupted images +curl 'http://localhost:3001/api/images?corrupted=true' + +# Re-fetch if source still available +POST /api/images with same source_url +``` + +## Advanced Usage + +### Export Dataset for ML Training + +```bash +# Get all images with specific tag +curl 'http://localhost:3001/api/images?tag=weather-sunny&pageSize=999' \ + -o dataset.json +``` + +### Monitor Fetch Status + +```bash +curl http://localhost:3001/api/fetcher/status +``` + +### Batch Operations + +```bash +# Add tags to multiple images (via API loop) +for image_id in 1 2 3 4 5; do + curl -X POST http://localhost:3001/api/images/$image_id/tags \ + -H "Content-Type: application/json" \ + -d '{"tags": ["batch-import"]}' +done +``` + +## Security Considerations + +1. **Database Access**: SQLite database is file-based; protect with filesystem permissions +2. **Image Storage**: Protect `data/images` directory from unauthorized access +3. **API Security**: Consider adding authentication for production use +4. **File Validation**: System validates MIME types and file sizes + +## Performance Metrics + +- **Fetch Time**: ~1-5 seconds per image (network dependent) +- **Database Queries**: <100ms for typical queries +- **Verification**: ~50ms per image +- **Storage**: ~1KB overhead per image in database + +--- + +For more information, check the API documentation or run `node setup.js help`. diff --git a/README.md b/README.md index ab1ce5e..c013aa1 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,18 @@ # HomeBase -Simple self-hosted Node.js Docker application with automated SSH deployment. +Self-hosted Node.js Docker application with automated SSH deployment and **long-term image storage** with corruption detection and tagging for ML training. ## Features -- ๐Ÿš€ Simple Express.js server -- ๐Ÿณ Docker containerized +- ๐Ÿš€ Express.js server +- ๐Ÿณ Docker containerized with persistent volumes - ๐Ÿ”„ Automated deployment via SSH -- โœ… Health check endpoint +- ๐Ÿ“ธ **Long-term image storage** with SQLite database +- ๐Ÿท๏ธ **Tag-based organization** for machine learning datasets +- ๐Ÿ›ก๏ธ **Corruption detection** using SHA256 checksums +- ๐Ÿ” **RESTful API** for image management +- ๐Ÿงช **Automatic fetching** every 2-3 minutes from web services +- โœ… Health check endpoints - ๐Ÿ” SSH key-based authentication ## Prerequisites @@ -30,53 +35,70 @@ On the remote server (beepc): # Install dependencies npm install +# Initialize the image storage system +node setup.js init + +# Configure image sources (edit image-sources.json) +# Set "enabled": true for sources you want to fetch from + # Run locally npm start # Visit http://localhost:3001 ``` +### Image Storage Features + +Once running, manage images via the REST API: + +```bash +# List all images +curl http://localhost:3001/api/images + +# Add an image manually +curl -X POST http://localhost:3001/api/images \ + -H "Content-Type: application/json" \ + -d '{"source_url":"https://example.com/image.jpg","tags":["test"]}' + +# Get storage statistics +curl http://localhost:3001/api/stats + +# Verify all images for corruption +curl -X POST http://localhost:3001/api/verify +``` + +๐Ÿ“– **Full documentation:** +- [๐Ÿ“ธ Image Storage Guide](IMAGE_STORAGE_GUIDE.md) - Complete API reference, configuration, and usage +- [๐Ÿณ Docker Deployment Guide](DOCKER_DEPLOYMENT.md) - Deployment to home lab with persistent storage + ### Deployment -### Option 1: Manual Deployment (Windows) +### Quick Deployment to Home Lab (PowerShell) -```bash -# Run the deployment script from project root -.\scripts\deploy.bat -``` - -### Option 2: Manual Deployment (Linux/Mac) - -```bash -# Make script executable -chmod +x scripts/deploy.sh - -# Run deployment -bash scripts/deploy.sh -``` - -### Option 3: Manual Deployment (PowerShell) +From your local Windows machine: ```powershell -# Run the deployment script from project root +# Deploy to home lab (beepc) .\scripts\deploy.ps1 ``` -### Option 4: Automated Deployment (GitHub Actions) +This will: +1. Copy all project files to your home lab server +2. Build the Docker image +3. Create and start the container with persistent storage +4. Set up systemd service for auto-start -1. Add your SSH private key to GitHub Secrets: - - Go to your repository Settings > Secrets and variables > Actions - - Add a new secret named `SSH_PRIVATE_KEY` - - Paste your SSH private key content +The app will be available at `http://homebase.sketchferret.com` -2. Push to main branch: - ```bash - git add . - git commit -m "Initial commit" - git push origin main - ``` +### Persistent Data Storage -3. The app will automatically deploy to spencer@beepc +Images and database are stored in a Docker volume (`homebase-data`) that persists across container restarts and updates. + +For detailed deployment instructions, see [DOCKER_DEPLOYMENT.md](DOCKER_DEPLOYMENT.md) + +### Option 2: Manual Deployment + +Manual deployment steps are documented in [DOCKER_DEPLOYMENT.md](DOCKER_DEPLOYMENT.md#manual-updates) ## SSH Setup @@ -114,28 +136,62 @@ sudo apt-get install docker-compose-plugin ## API Endpoints -- `GET /` - Main endpoint, returns app status +### Status & Health +- `GET /api/status` - App status and version - `GET /health` - Health check endpoint +### Image Management +- `GET /api/images` - List all images (with pagination & filtering) +- `GET /api/images/{id}` - Get image details +- `GET /api/images/{id}/download` - Download image file +- `POST /api/images` - Fetch and save new image +- `DELETE /api/images/{id}` - Delete image + +### Tagging +- `POST /api/images/{id}/tags` - Add tags to image +- `GET /api/tags` - List all available tags + +### Administration +- `GET /api/stats` - Storage statistics +- `POST /api/verify` - Verify all images for corruption +- `POST /api/cleanup` - Clean up old images +- `GET /api/fetcher/status` - Image fetcher status + +See [IMAGE_STORAGE_GUIDE.md](IMAGE_STORAGE_GUIDE.md) for detailed API documentation. + ## Project Structure ``` HomeBase/ โ”œโ”€โ”€ scripts/ -โ”‚ โ”œโ”€โ”€ deploy.ps1 # PowerShell deployment script -โ”‚ โ”œโ”€โ”€ deploy.sh # Bash deployment script -โ”‚ โ””โ”€โ”€ deploy.bat # Windows batch deployment script +โ”‚ โ”œโ”€โ”€ deploy.ps1 # PowerShell deployment script +โ”‚ โ”œโ”€โ”€ deploy.sh # Bash deployment script +โ”‚ โ””โ”€โ”€ deploy.bat # Windows batch deployment script +โ”œโ”€โ”€ lib/ +โ”‚ โ”œโ”€โ”€ database.js # SQLite operations & schema +โ”‚ โ”œโ”€โ”€ storage.js # File storage & checksums +โ”‚ โ””โ”€โ”€ image-fetcher.js # Image fetching service +โ”œโ”€โ”€ routes/ +โ”‚ โ””โ”€โ”€ images.js # Image API endpoints +โ”œโ”€โ”€ public/ +โ”‚ โ”œโ”€โ”€ index.html +โ”‚ โ”œโ”€โ”€ app.js +โ”‚ โ””โ”€โ”€ styles.css โ”œโ”€โ”€ .github/ โ”‚ โ””โ”€โ”€ workflows/ -โ”‚ โ””โ”€โ”€ deploy.yml # GitHub Actions workflow -โ”œโ”€โ”€ .dockerignore # Docker ignore file -โ”œโ”€โ”€ .gitignore # Git ignore file -โ”œโ”€โ”€ Dockerfile # Docker configuration -โ”œโ”€โ”€ docker-compose.yml # Docker Compose configuration -โ”œโ”€โ”€ homebase.service # Systemd service file -โ”œโ”€โ”€ package.json # Node.js dependencies -โ”œโ”€โ”€ server.js # Express server -โ””โ”€โ”€ README.md # This file +โ”‚ โ””โ”€โ”€ deploy.yml # GitHub Actions workflow +โ”œโ”€โ”€ .dockerignore # Docker ignore file +โ”œโ”€โ”€ .gitignore # Git ignore file +โ”œโ”€โ”€ Dockerfile # Docker configuration +โ”œโ”€โ”€ docker-compose.yml # Docker Compose with volumes +โ”œโ”€โ”€ homebase.service # Systemd service file +โ”œโ”€โ”€ package.json # Node.js dependencies +โ”œโ”€โ”€ server.js # Express server +โ”œโ”€โ”€ setup.js # Setup & CLI tool +โ”œโ”€โ”€ image-sources.json # Image source configuration +โ”œโ”€โ”€ IMAGE_STORAGE_GUIDE.md # Complete image storage docs +โ”œโ”€โ”€ DOCKER_DEPLOYMENT.md # Docker deployment guide +โ””โ”€โ”€ README.md # This file ``` ## Deployment Process @@ -212,16 +268,44 @@ ssh spencer@beepc "sudo systemctl restart homebase" ssh spencer@beepc "sudo systemctl disable homebase" ``` -## Environment Variables +## Configuration -You can customize settings by modifying [docker-compose.yml](docker-compose.yml): +### Image Sources + +Edit `image-sources.json` to configure which image URLs to fetch from: + +```json +{ + "sources": [ + { + "name": "Webcam Feed", + "url": "https://your-service.com/image.jpg", + "tags": ["webcam", "monitoring"], + "enabled": true + } + ], + "fetchInterval": 0.033 +} +``` + +**Fetch Intervals** - Edit `fetchInterval` (in minutes): +- `0.0167` = 1 second +- `0.033` = 2 seconds (default, recommended for steady updates) +- `0.05` = 3 seconds +- `1` = 1 minute +- `2.5` = 2.5 minutes (original default) + +The fetcher will automatically start pulling images from enabled sources at this interval. + +### Environment Variables + +Customize settings in [docker-compose.yml](docker-compose.yml): ```yaml environment: - - PORT=3001 # Internal port (reverse proxy maps to standard HTTP) - - DOMAIN=homebase.sketchferret.com # Your domain -ports: - - "3001:3001" # Port mapping for reverse proxy backend + - PORT=3001 + - DOMAIN=homebase.sketchferret.com + - NODE_ENV=production ``` ## Security Notes diff --git a/docker-compose.yml b/docker-compose.yml index ed0b4b9..247921c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,9 +3,19 @@ services: build: . container_name: homebase restart: unless-stopped - ports: - - "3001:3001" + # Use host network to access both local network and Tailscale + # Also inherits host's DNS for .local mDNS hostname resolution + network_mode: host environment: - NODE_ENV=production - PORT=3001 - DOMAIN=homebase.sketchferret.com + volumes: + # Persist database and image files + - homebase-data:/app/data + # Mount host's resolv.conf for mDNS support + - /etc/resolv.conf:/etc/resolv.conf:ro + +volumes: + homebase-data: + driver: local diff --git a/image-sources.json b/image-sources.json new file mode 100644 index 0000000..3941cd4 --- /dev/null +++ b/image-sources.json @@ -0,0 +1,19 @@ +{ + "sources": [ + { + "name": "Bedroom Exterior", + "url": "http://192.168.254.66:8080/", + "tags": ["lake", "road"], + "enabled": true + } + ], + "fetchInterval": 0.033, + "comments": { + "sources": "Array of image sources to fetch from", + "name": "Human-readable name for the source", + "url": "Full URL to the image", + "tags": "Array of tags to apply to fetched images (useful for ML training)", + "enabled": "Set to true to include this source in fetching", + "fetchInterval": "Minutes between fetch cycles. Examples: 0.033 = 2 seconds, 0.05 = 3 seconds, 0.083 = 5 seconds, 1 = 1 minute, 2.5 = 2.5 minutes (default)" + } +} diff --git a/lib/database.js b/lib/database.js new file mode 100644 index 0000000..0476825 --- /dev/null +++ b/lib/database.js @@ -0,0 +1,342 @@ +const sqlite3 = require('sqlite3').verbose(); +const path = require('path'); +const fs = require('fs'); + +const DB_PATH = path.join(__dirname, '..', 'data', 'homebase.db'); + +// Ensure data directory exists +const dataDir = path.dirname(DB_PATH); +if (!fs.existsSync(dataDir)) { + fs.mkdirSync(dataDir, { recursive: true }); +} + +const db = new sqlite3.Database(DB_PATH, (err) => { + if (err) { + console.error('Database connection error:', err); + } else { + console.log('Connected to SQLite database at', DB_PATH); + } +}); + +// Enable foreign keys +db.run('PRAGMA foreign_keys = ON'); + +/** + * Initialize database schema + */ +async function initializeDatabase() { + return new Promise((resolve, reject) => { + db.serialize(() => { + // Images table + db.run( + `CREATE TABLE IF NOT EXISTS images ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT NOT NULL UNIQUE, + source_url TEXT NOT NULL, + file_path TEXT NOT NULL, + filesize INTEGER, + file_hash TEXT NOT NULL, + mime_type TEXT, + fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + is_corrupted BOOLEAN DEFAULT 0 + )`, + (err) => { + if (err) reject(err); + } + ); + + // Tags table + db.run( + `CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + image_id INTEGER NOT NULL, + tag TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (image_id) REFERENCES images(id) ON DELETE CASCADE, + UNIQUE(image_id, tag) + )`, + (err) => { + if (err) reject(err); + } + ); + + // Create indexes for better query performance + db.run('CREATE INDEX IF NOT EXISTS idx_images_source_url ON images(source_url)'); + db.run('CREATE INDEX IF NOT EXISTS idx_images_fetched_at ON images(fetched_at)'); + db.run('CREATE INDEX IF NOT EXISTS idx_images_file_hash ON images(file_hash)'); + db.run('CREATE INDEX IF NOT EXISTS idx_tags_image_id ON tags(image_id)'); + db.run('CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag)', (err) => { + if (err) { + reject(err); + } else { + console.log('Database initialized successfully'); + resolve(); + } + }); + }); + }); +} + +/** + * Insert a new image record + */ +function insertImage(imageData) { + return new Promise((resolve, reject) => { + const { filename, source_url, file_path, filesize, file_hash, mime_type } = imageData; + + db.run( + `INSERT INTO images (filename, source_url, file_path, filesize, file_hash, mime_type) + VALUES (?, ?, ?, ?, ?, ?)`, + [filename, source_url, file_path, filesize, file_hash, mime_type], + function(err) { + if (err) { + reject(err); + } else { + resolve({ id: this.lastID }); + } + } + ); + }); +} + +/** + * Add tags to an image + */ +function addTags(imageId, tags) { + return new Promise((resolve, reject) => { + if (!Array.isArray(tags) || tags.length === 0) { + resolve(); + return; + } + + db.serialize(() => { + const stmt = db.prepare( + 'INSERT OR IGNORE INTO tags (image_id, tag) VALUES (?, ?)' + ); + + let completed = 0; + let hasError = false; + + tags.forEach((tag) => { + stmt.run([imageId, tag], (err) => { + if (err && !hasError) { + hasError = true; + stmt.finalize(); + reject(err); + } else { + completed++; + if (completed === tags.length) { + stmt.finalize((err) => { + if (err) reject(err); + else resolve(); + }); + } + } + }); + }); + }); + }); +} + +/** + * Mark image as corrupted + */ +function markAsCorrupted(imageId) { + return new Promise((resolve, reject) => { + db.run( + 'UPDATE images SET is_corrupted = 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?', + [imageId], + (err) => { + if (err) reject(err); + else resolve(); + } + ); + }); +} + +/** + * Get image by ID + */ +function getImage(imageId) { + return new Promise((resolve, reject) => { + db.get( + 'SELECT * FROM images WHERE id = ?', + [imageId], + (err, row) => { + if (err) reject(err); + else resolve(row); + } + ); + }); +} + +/** + * Get all images with optional filtering + */ +function getImages(options = {}) { + return new Promise((resolve, reject) => { + const { tag, sourceUrl, limit = 100, offset = 0, sortBy = 'fetched_at', order = 'DESC' } = options; + + let query = 'SELECT DISTINCT i.* FROM images i'; + let params = []; + + if (tag) { + query += ' INNER JOIN tags t ON i.id = t.image_id'; + } + + query += ' WHERE i.is_corrupted = 0'; + + if (tag) { + query += ' AND t.tag = ?'; + params.push(tag); + } + + if (sourceUrl) { + query += ' AND i.source_url = ?'; + params.push(sourceUrl); + } + + query += ` ORDER BY i.${sortBy} ${order} LIMIT ? OFFSET ?`; + params.push(limit, offset); + + db.all(query, params, (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + }); + }); +} + +/** + * Get tags for an image + */ +function getImageTags(imageId) { + return new Promise((resolve, reject) => { + db.all( + 'SELECT tag FROM tags WHERE image_id = ? ORDER BY tag', + [imageId], + (err, rows) => { + if (err) reject(err); + else resolve((rows || []).map(r => r.tag)); + } + ); + }); +} + +/** + * Get all unique tags + */ +function getAllTags() { + return new Promise((resolve, reject) => { + db.all( + 'SELECT DISTINCT tag FROM tags ORDER BY tag', + (err, rows) => { + if (err) reject(err); + else resolve((rows || []).map(r => r.tag)); + } + ); + }); +} + +/** + * Get image count + */ +function getImageCount(tag = null) { + return new Promise((resolve, reject) => { + let query = 'SELECT COUNT(*) as count FROM images WHERE is_corrupted = 0'; + let params = []; + + if (tag) { + query = `SELECT COUNT(DISTINCT i.id) as count FROM images i + INNER JOIN tags t ON i.id = t.image_id + WHERE i.is_corrupted = 0 AND t.tag = ?`; + params.push(tag); + } + + db.get(query, params, (err, row) => { + if (err) reject(err); + else resolve(row?.count || 0); + }); + }); +} + +/** + * Delete image (soft delete - keeps file but marks for cleanup) + */ +function deleteImage(imageId) { + return new Promise((resolve, reject) => { + db.run( + 'DELETE FROM images WHERE id = ?', + [imageId], + (err) => { + if (err) reject(err); + else resolve(); + } + ); + }); +} + +/** + * Clean up old images (older than specified days) + */ +function cleanupOldImages(daysOld = 30) { + return new Promise((resolve, reject) => { + db.run( + `DELETE FROM images WHERE fetched_at < datetime('now', '-' || ? || ' days')`, + [daysOld], + function(err) { + if (err) reject(err); + else resolve(this.changes); + } + ); + }); +} + +/** + * Get images by hash (detect duplicates) + */ +function getImagesByHash(hash) { + return new Promise((resolve, reject) => { + db.all( + 'SELECT * FROM images WHERE file_hash = ? AND is_corrupted = 0', + [hash], + (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + } + ); + }); +} + +/** + * Close database connection + */ +function closeDatabase() { + return new Promise((resolve, reject) => { + db.close((err) => { + if (err) reject(err); + else { + console.log('Database connection closed'); + resolve(); + } + }); + }); +} + +module.exports = { + db, + initializeDatabase, + insertImage, + addTags, + markAsCorrupted, + getImage, + getImages, + getImageTags, + getAllTags, + getImageCount, + deleteImage, + cleanupOldImages, + getImagesByHash, + closeDatabase +}; diff --git a/lib/image-fetcher.js b/lib/image-fetcher.js new file mode 100644 index 0000000..f04ea3a --- /dev/null +++ b/lib/image-fetcher.js @@ -0,0 +1,162 @@ +const storage = require('./storage'); +const database = require('./database'); +const axios = require('axios'); + +let fetchInterval = null; +let isRunning = false; + +/** + * Fetch a single image from URL + */ +async function fetchImage(sourceUrl, tags = []) { + try { + console.log(`[Image Fetcher] Fetching from: ${sourceUrl}`); + + // Download and save image + const downloadResult = await storage.downloadAndSaveImage(sourceUrl); + + if (!downloadResult.success) { + console.error(`[Image Fetcher] Failed to download: ${downloadResult.error}`); + return { success: false, error: downloadResult.error }; + } + + // Insert into database + const imageRecord = await database.insertImage({ + filename: downloadResult.filename, + source_url: sourceUrl, + file_path: downloadResult.file_path, + filesize: downloadResult.filesize, + file_hash: downloadResult.file_hash, + mime_type: downloadResult.mime_type + }); + + // Add tags if provided + if (tags && tags.length > 0) { + await database.addTags(imageRecord.id, tags); + } + + console.log(`[Image Fetcher] Saved image: ${downloadResult.filename} (ID: ${imageRecord.id})`); + + return { + success: true, + imageId: imageRecord.id, + filename: downloadResult.filename, + filesize: downloadResult.filesize + }; + } catch (err) { + console.error('[Image Fetcher] Error:', err.message); + return { success: false, error: err.message }; + } +} + +/** + * Verify all images in database for corruption + */ +async function verifyAllImages() { + try { + console.log('[Image Fetcher] Starting verification of all images...'); + + const images = await database.getImages({ limit: 10000 }); + let corruptCount = 0; + + for (const image of images) { + const verification = await storage.verifyImageIntegrity(image.file_path, image.file_hash); + + if (!verification.valid) { + console.warn(`[Image Fetcher] Corrupted image detected: ${image.filename} - ${verification.reason}`); + await database.markAsCorrupted(image.id); + corruptCount++; + } + } + + if (corruptCount > 0) { + console.log(`[Image Fetcher] Found and marked ${corruptCount} corrupted images`); + } else { + console.log('[Image Fetcher] Verification complete - all images intact'); + } + + return { checked: images.length, corrupted: corruptCount }; + } catch (err) { + console.error('[Image Fetcher] Verification error:', err.message); + } +} + +/** + * Fetch from multiple sources + */ +async function fetchFromMultipleSources(sources) { + console.log(`[Image Fetcher] Fetching from ${sources.length} source(s)...`); + + const results = []; + for (const source of sources) { + const sourceUrl = typeof source === 'string' ? source : source.url; + const tags = source.tags || []; + + const result = await fetchImage(sourceUrl, tags); + results.push(result); + } + + return results; +} + +/** + * Start periodic image fetching + */ +function startFetcher(sources, intervalMinutes = 2.5) { + if (isRunning) { + console.log('[Image Fetcher] Fetcher already running'); + return; + } + + isRunning = true; + const intervalMs = intervalMinutes * 60 * 1000; + + console.log(`[Image Fetcher] Starting fetcher with ${intervalMinutes} minute interval`); + console.log(`[Image Fetcher] Will fetch from ${sources.length} source(s)`); + + // Do initial fetch immediately + fetchFromMultipleSources(sources); + + // Then set up interval + fetchInterval = setInterval(() => { + fetchFromMultipleSources(sources); + }, intervalMs); + + return { + status: 'running', + interval: intervalMinutes, + sources: sources.length, + nextFetch: new Date(Date.now() + intervalMs) + }; +} + +/** + * Stop the fetcher + */ +function stopFetcher() { + if (fetchInterval) { + clearInterval(fetchInterval); + fetchInterval = null; + isRunning = false; + console.log('[Image Fetcher] Fetcher stopped'); + } +} + +/** + * Get fetcher status + */ +function getStatus() { + return { + running: isRunning, + nextFetch: fetchInterval ? 'scheduled' : 'not running' + }; +} + +module.exports = { + fetchImage, + fetchFromMultipleSources, + startFetcher, + stopFetcher, + verifyAllImages, + getStatus +}; diff --git a/lib/storage.js b/lib/storage.js new file mode 100644 index 0000000..077f8b5 --- /dev/null +++ b/lib/storage.js @@ -0,0 +1,244 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const axios = require('axios'); + +const IMAGES_DIR = path.join(__dirname, '..', 'data', 'images'); + +// Ensure images directory exists +if (!fs.existsSync(IMAGES_DIR)) { + fs.mkdirSync(IMAGES_DIR, { recursive: true }); +} + +/** + * Calculate SHA256 hash of a file + */ +function calculateFileHash(filePath) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash('sha256'); + const stream = fs.createReadStream(filePath); + + stream.on('error', reject); + stream.on('data', chunk => hash.update(chunk)); + stream.on('end', () => resolve(hash.digest('hex'))); + }); +} + +/** + * Generate a unique filename + */ +function generateFilename(sourceUrl, timestamp = Date.now()) { + const hash = crypto.createHash('md5').update(sourceUrl + timestamp).digest('hex'); + const ext = getFileExtension(sourceUrl); + return `image_${timestamp}_${hash}${ext}`; +} + +/** + * Get file extension from URL + */ +function getFileExtension(url) { + try { + const urlObj = new URL(url); + const pathname = urlObj.pathname; + const ext = path.extname(pathname); + + // If no extension, try to get from content-type or use default + if (!ext || ext.length > 5) { + return '.jpg'; + } + return ext; + } catch (err) { + return '.jpg'; + } +} + +/** + * Download an image and save it to storage + */ +async function downloadAndSaveImage(sourceUrl, options = {}) { + const { timeout = 30000, maxSize = 50 * 1024 * 1024 } = options; + + try { + // Download image + const response = await axios({ + url: sourceUrl, + method: 'GET', + responseType: 'arraybuffer', + timeout, + maxContentLength: maxSize, + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' + } + }); + + const filename = generateFilename(sourceUrl); + const filePath = path.join(IMAGES_DIR, filename); + const fileBuffer = Buffer.from(response.data); + const filesize = fileBuffer.length; + const file_hash = crypto.createHash('sha256').update(fileBuffer).digest('hex'); + const mime_type = response.headers['content-type'] || 'image/jpeg'; + + // Save file + fs.writeFileSync(filePath, fileBuffer); + + return { + filename, + file_path: filePath, + relative_path: path.relative(path.join(__dirname, '..'), filePath), + filesize, + file_hash, + mime_type, + source_url: sourceUrl, + success: true + }; + } catch (err) { + console.error(`Failed to download image from ${sourceUrl}:`, err.message); + return { + source_url: sourceUrl, + success: false, + error: err.message + }; + } +} + +/** + * Verify image integrity using stored hash + */ +async function verifyImageIntegrity(filePath, expectedHash) { + try { + if (!fs.existsSync(filePath)) { + return { valid: false, reason: 'File does not exist' }; + } + + const actualHash = await calculateFileHash(filePath); + + if (actualHash !== expectedHash) { + return { valid: false, reason: 'Hash mismatch', expectedHash, actualHash }; + } + + return { valid: true }; + } catch (err) { + return { valid: false, reason: err.message }; + } +} + +/** + * Delete image file + */ +function deleteImageFile(filePath) { + return new Promise((resolve, reject) => { + try { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + resolve(true); + } else { + resolve(false); + } + } catch (err) { + reject(err); + } + }); +} + +/** + * Get image file as buffer + */ +function getImageBuffer(filePath) { + return new Promise((resolve, reject) => { + fs.readFile(filePath, (err, data) => { + if (err) reject(err); + else resolve(data); + }); + }); +} + +/** + * Get storage statistics + */ +function getStorageStats() { + return new Promise((resolve, reject) => { + try { + let totalSize = 0; + let fileCount = 0; + + if (!fs.existsSync(IMAGES_DIR)) { + resolve({ totalSize: 0, fileCount: 0, dataDir: IMAGES_DIR }); + return; + } + + const files = fs.readdirSync(IMAGES_DIR); + + files.forEach((file) => { + const filePath = path.join(IMAGES_DIR, file); + const stats = fs.statSync(filePath); + if (stats.isFile()) { + totalSize += stats.size; + fileCount++; + } + }); + + resolve({ + totalSize, + totalSizeGB: (totalSize / (1024 ** 3)).toFixed(2), + fileCount, + dataDir: IMAGES_DIR + }); + } catch (err) { + reject(err); + } + }); +} + +/** + * Clean up orphaned files (files in storage but not in database) + */ +async function cleanupOrphanedFiles(db) { + try { + if (!fs.existsSync(IMAGES_DIR)) { + return { cleaned: 0 }; + } + + const files = fs.readdirSync(IMAGES_DIR); + let cleaned = 0; + + for (const file of files) { + const filePath = path.join(IMAGES_DIR, file); + + // Check if file exists in database + const existsInDb = await new Promise((resolve) => { + db.get( + 'SELECT id FROM images WHERE filename = ?', + [file], + (err, row) => resolve(!err && !!row) + ); + }); + + if (!existsInDb) { + try { + fs.unlinkSync(filePath); + cleaned++; + } catch (err) { + console.error(`Failed to delete orphaned file ${file}:`, err.message); + } + } + } + + return { cleaned }; + } catch (err) { + console.error('Cleanup error:', err); + return { cleaned: 0, error: err.message }; + } +} + +module.exports = { + downloadAndSaveImage, + verifyImageIntegrity, + deleteImageFile, + getImageBuffer, + getStorageStats, + calculateFileHash, + generateFilename, + getFileExtension, + cleanupOrphanedFiles, + IMAGES_DIR +}; diff --git a/package.json b/package.json index 2909430..0cee85a 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,9 @@ "author": "", "license": "MIT", "dependencies": { - "express": "^4.18.2" + "express": "^4.18.2", + "sqlite3": "^5.1.6", + "axios": "^1.6.0", + "crypto": "^1.0.1" } } diff --git a/public/gallery.html b/public/gallery.html new file mode 100644 index 0000000..38b4b32 --- /dev/null +++ b/public/gallery.html @@ -0,0 +1,587 @@ + + + + + + HomeBase Image Gallery + + + +
+
+

๐Ÿ“ธ Image Timeline

+

Browse your captured images chronologically

+
+ +
+ + + +
+ +
+
+ Total Images + 0 +
+
+ Total Storage + 0 GB +
+
+ Last Updated + --:-- +
+
+ +
Loading images...
+ + + +
+ + + + + + + diff --git a/public/index.html b/public/index.html index 48a0496..3f2c4f7 100644 --- a/public/index.html +++ b/public/index.html @@ -36,6 +36,27 @@ + +
+

Features

+
+ +
๐Ÿ“ธ
+

Image Gallery

+

Browse captured images on an interactive timeline

+ + +
๐Ÿ“Š
+

Statistics

+

View storage usage and image metrics

+ + +
โœจ
+

API Status

+

Check system status and features

+ +
+