feat: Implement image fetching and storage system
Some checks failed
Deploy to BeePC / deploy (push) Has been cancelled
Some checks failed
Deploy to BeePC / deploy (push) Has been cancelled
- Add image-fetcher module for downloading and saving images from various sources. - Create storage module for managing image files, including downloading, verifying integrity, and cleaning up orphaned files. - Develop gallery HTML page for displaying images with sorting and filtering options. - Set up RESTful API routes for image management, including fetching, adding tags, and deleting images. - Introduce setup script for initializing the database and configuring image sources. - Implement a batch process for verifying image integrity and cleaning up old images. - Add setup batch script for easy installation and configuration of the image storage system.
This commit is contained in:
342
lib/database.js
Normal file
342
lib/database.js
Normal file
@@ -0,0 +1,342 @@
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const DB_PATH = path.join(__dirname, '..', 'data', 'homebase.db');
|
||||
|
||||
// Ensure data directory exists
|
||||
const dataDir = path.dirname(DB_PATH);
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
const db = new sqlite3.Database(DB_PATH, (err) => {
|
||||
if (err) {
|
||||
console.error('Database connection error:', err);
|
||||
} else {
|
||||
console.log('Connected to SQLite database at', DB_PATH);
|
||||
}
|
||||
});
|
||||
|
||||
// Enable foreign keys
|
||||
db.run('PRAGMA foreign_keys = ON');
|
||||
|
||||
/**
|
||||
* Initialize database schema
|
||||
*/
|
||||
async function initializeDatabase() {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.serialize(() => {
|
||||
// Images table
|
||||
db.run(
|
||||
`CREATE TABLE IF NOT EXISTS images (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
filename TEXT NOT NULL UNIQUE,
|
||||
source_url TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
filesize INTEGER,
|
||||
file_hash TEXT NOT NULL,
|
||||
mime_type TEXT,
|
||||
fetched_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
is_corrupted BOOLEAN DEFAULT 0
|
||||
)`,
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
}
|
||||
);
|
||||
|
||||
// Tags table
|
||||
db.run(
|
||||
`CREATE TABLE IF NOT EXISTS tags (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
image_id INTEGER NOT NULL,
|
||||
tag TEXT NOT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (image_id) REFERENCES images(id) ON DELETE CASCADE,
|
||||
UNIQUE(image_id, tag)
|
||||
)`,
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
}
|
||||
);
|
||||
|
||||
// Create indexes for better query performance
|
||||
db.run('CREATE INDEX IF NOT EXISTS idx_images_source_url ON images(source_url)');
|
||||
db.run('CREATE INDEX IF NOT EXISTS idx_images_fetched_at ON images(fetched_at)');
|
||||
db.run('CREATE INDEX IF NOT EXISTS idx_images_file_hash ON images(file_hash)');
|
||||
db.run('CREATE INDEX IF NOT EXISTS idx_tags_image_id ON tags(image_id)');
|
||||
db.run('CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag)', (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
console.log('Database initialized successfully');
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a new image record
|
||||
*/
|
||||
function insertImage(imageData) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const { filename, source_url, file_path, filesize, file_hash, mime_type } = imageData;
|
||||
|
||||
db.run(
|
||||
`INSERT INTO images (filename, source_url, file_path, filesize, file_hash, mime_type)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
[filename, source_url, file_path, filesize, file_hash, mime_type],
|
||||
function(err) {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
resolve({ id: this.lastID });
|
||||
}
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Add tags to an image
|
||||
*/
|
||||
function addTags(imageId, tags) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!Array.isArray(tags) || tags.length === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
db.serialize(() => {
|
||||
const stmt = db.prepare(
|
||||
'INSERT OR IGNORE INTO tags (image_id, tag) VALUES (?, ?)'
|
||||
);
|
||||
|
||||
let completed = 0;
|
||||
let hasError = false;
|
||||
|
||||
tags.forEach((tag) => {
|
||||
stmt.run([imageId, tag], (err) => {
|
||||
if (err && !hasError) {
|
||||
hasError = true;
|
||||
stmt.finalize();
|
||||
reject(err);
|
||||
} else {
|
||||
completed++;
|
||||
if (completed === tags.length) {
|
||||
stmt.finalize((err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark image as corrupted
|
||||
*/
|
||||
function markAsCorrupted(imageId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(
|
||||
'UPDATE images SET is_corrupted = 1, updated_at = CURRENT_TIMESTAMP WHERE id = ?',
|
||||
[imageId],
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get image by ID
|
||||
*/
|
||||
function getImage(imageId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get(
|
||||
'SELECT * FROM images WHERE id = ?',
|
||||
[imageId],
|
||||
(err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all images with optional filtering
|
||||
*/
|
||||
function getImages(options = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const { tag, sourceUrl, limit = 100, offset = 0, sortBy = 'fetched_at', order = 'DESC' } = options;
|
||||
|
||||
let query = 'SELECT DISTINCT i.* FROM images i';
|
||||
let params = [];
|
||||
|
||||
if (tag) {
|
||||
query += ' INNER JOIN tags t ON i.id = t.image_id';
|
||||
}
|
||||
|
||||
query += ' WHERE i.is_corrupted = 0';
|
||||
|
||||
if (tag) {
|
||||
query += ' AND t.tag = ?';
|
||||
params.push(tag);
|
||||
}
|
||||
|
||||
if (sourceUrl) {
|
||||
query += ' AND i.source_url = ?';
|
||||
params.push(sourceUrl);
|
||||
}
|
||||
|
||||
query += ` ORDER BY i.${sortBy} ${order} LIMIT ? OFFSET ?`;
|
||||
params.push(limit, offset);
|
||||
|
||||
db.all(query, params, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows || []);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get tags for an image
|
||||
*/
|
||||
function getImageTags(imageId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(
|
||||
'SELECT tag FROM tags WHERE image_id = ? ORDER BY tag',
|
||||
[imageId],
|
||||
(err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve((rows || []).map(r => r.tag));
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all unique tags
|
||||
*/
|
||||
function getAllTags() {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(
|
||||
'SELECT DISTINCT tag FROM tags ORDER BY tag',
|
||||
(err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve((rows || []).map(r => r.tag));
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get image count
|
||||
*/
|
||||
function getImageCount(tag = null) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let query = 'SELECT COUNT(*) as count FROM images WHERE is_corrupted = 0';
|
||||
let params = [];
|
||||
|
||||
if (tag) {
|
||||
query = `SELECT COUNT(DISTINCT i.id) as count FROM images i
|
||||
INNER JOIN tags t ON i.id = t.image_id
|
||||
WHERE i.is_corrupted = 0 AND t.tag = ?`;
|
||||
params.push(tag);
|
||||
}
|
||||
|
||||
db.get(query, params, (err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row?.count || 0);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete image (soft delete - keeps file but marks for cleanup)
|
||||
*/
|
||||
function deleteImage(imageId) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(
|
||||
'DELETE FROM images WHERE id = ?',
|
||||
[imageId],
|
||||
(err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old images (older than specified days)
|
||||
*/
|
||||
function cleanupOldImages(daysOld = 30) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.run(
|
||||
`DELETE FROM images WHERE fetched_at < datetime('now', '-' || ? || ' days')`,
|
||||
[daysOld],
|
||||
function(err) {
|
||||
if (err) reject(err);
|
||||
else resolve(this.changes);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get images by hash (detect duplicates)
|
||||
*/
|
||||
function getImagesByHash(hash) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(
|
||||
'SELECT * FROM images WHERE file_hash = ? AND is_corrupted = 0',
|
||||
[hash],
|
||||
(err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows || []);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Close database connection
|
||||
*/
|
||||
function closeDatabase() {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.close((err) => {
|
||||
if (err) reject(err);
|
||||
else {
|
||||
console.log('Database connection closed');
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
db,
|
||||
initializeDatabase,
|
||||
insertImage,
|
||||
addTags,
|
||||
markAsCorrupted,
|
||||
getImage,
|
||||
getImages,
|
||||
getImageTags,
|
||||
getAllTags,
|
||||
getImageCount,
|
||||
deleteImage,
|
||||
cleanupOldImages,
|
||||
getImagesByHash,
|
||||
closeDatabase
|
||||
};
|
||||
Reference in New Issue
Block a user