jmb/48hr.email
1
0
Fork 0
mirror of https://github.com/Crazyco-xyz/48hr.email.git synced 2026-02-14 17:19:35 +01:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

base: jmb:51d608c2c00b39e939fa7d51c0d12976db42756e
Branches Tags
jmb:main
..
compare: jmb:f8ce98a5846567298e800dcb532c73224fb08e06
Branches Tags
jmb:main

No commits in common. "51d608c2c00b39e939fa7d51c0d12976db42756e" and "f8ce98a5846567298e800dcb532c73224fb08e06" have entirely different histories.
51d608c2c0 ... f8ce98a584

2 changed files with 39 additions and 146 deletions
Show stats Expand all files Collapse all files

126
application/imap-service.js
View file

@ -253,46 +253,17 @@ class ImapService extends EventEmitter {
// IMAP date filters are unreliable - some servers search internal date, not Date header // IMAP date filters are unreliable - some servers search internal date, not Date header
// Always fetch all UIDs and filter by date header in JavaScript instead // Always fetch all UIDs and filter by date header in JavaScript instead
const deleteOlderThan = helper.purgeTimeStamp();
const searchQuery = [ const searchQuery = [
['!DELETED'] ['!DELETED']
]; ];
uids = await this._searchWithoutFetch(searchQuery); uids = await this._searchWithoutFetch(searchQuery);
if (uids.length === 0) return; if (uids.length === 0) return;
debug(`Starting deleteOldMails. Total UIDs: ${uids.length}`); const deleteOlderThan = helper.purgeTimeStamp();
const HEADER_BATCH_SIZE = this.config.imap.fetchChunkSize const exampleUids = this.config.email.examples.uids.map(x => parseInt(x));
const concurrency = this.config.imap.fetchConcurrency || 4; const headers = await this._getMailHeaders(uids);
// Chunk UIDs
const uidChunks = [];
for (let i = 0; i < uids.length; i += HEADER_BATCH_SIZE) {
uidChunks.push(uids.slice(i, i + HEADER_BATCH_SIZE));
}
let headers = [];
let workerId = 0;
const runNext = async() => {
if (workerId >= uidChunks.length) return;
const chunkId = workerId++;
const batch = uidChunks[chunkId];
try {
const batchHeaders = await Promise.race([
this._getMailHeaders(batch),
new Promise((_, reject) => setTimeout(() => reject(new Error('Header fetch timeout')), 30000))
]);
headers = headers.concat(batchHeaders);
} catch (err) {
debug(`ERROR in batch ${chunkId+1}: ${err.message}`);
}
await runNext();
};
// Start workers
const pool = [];
const workers = Math.min(concurrency, uidChunks.length);
for (let i = 0; i < workers; i++) {
pool.push(runNext());
}
await Promise.all(pool);
// Get locked inboxes if available // Get locked inboxes if available
let lockedAddresses = []; let lockedAddresses = [];
@ -305,26 +276,19 @@ class ImapService extends EventEmitter {
} }
} }
// Ensure exampleUids is defined before filtering
const exampleUids = this.config.email.examples && this.config.email.examples.uids ?
this.config.email.examples.uids.map(x => parseInt(x)) : [];
// Filter out mails that are too new, whitelisted, or belong to locked inboxes // Filter out mails that are too new, whitelisted, or belong to locked inboxes
let filtered = headers; const toDelete = headers
.filter(mail => {
const date = mail.attributes.date;
const uid = parseInt(mail.attributes.uid);
const toAddresses = Array.isArray(mail.parts[0].body.to) ?
mail.parts[0].body.to.map(a => a.toLowerCase()) : [String(mail.parts[0].body.to).toLowerCase()];
// Exclude exampleUids if (exampleUids.includes(uid)) return false;
filtered = filtered.filter(mail => !exampleUids.includes(parseInt(mail.attributes.uid))); if (toAddresses.some(addr => lockedAddresses.includes(addr))) return false;
return date <= deleteOlderThan;
// Exclude locked inboxes })
filtered = filtered.filter(mail => { .map(mail => parseInt(mail.attributes.uid));
const toAddresses = Array.isArray(mail.parts[0].body.to) ?
mail.parts[0].body.to.map(a => a.toLowerCase()) : [String(mail.parts[0].body.to).toLowerCase()];
return !toAddresses.some(addr => lockedAddresses.includes(addr));
});
// Exclude too new (compare timestamps for Date objects)
filtered = filtered.filter(mail => mail.attributes.date && mail.attributes.date.getTime() <= deleteOlderThan.getTime());
const toDelete = filtered.map(mail => parseInt(mail.attributes.uid));
if (toDelete.length === 0) { if (toDelete.length === 0) {
debug('No mails to delete. (after locked inbox exclusion)'); debug('No mails to delete. (after locked inbox exclusion)');
@ -332,15 +296,11 @@ class ImapService extends EventEmitter {
} }
debug(`Deleting mails ${toDelete}`); debug(`Deleting mails ${toDelete}`);
// Batch deletes to avoid IMAP argument limits await this.connection.deleteMessage(toDelete);
const BATCH_SIZE = this.config.imap.fetchChunkSize;
for (let i = 0; i < toDelete.length; i += BATCH_SIZE) { toDelete.forEach(uid => {
const batch = toDelete.slice(i, i + BATCH_SIZE); this.emit(ImapService.EVENT_DELETED_MAIL, uid);
await this.connection.deleteMessage(batch); });
batch.forEach(uid => {
this.emit(ImapService.EVENT_DELETED_MAIL, uid);
});
}
} }
@ -378,39 +338,17 @@ class ImapService extends EventEmitter {
* @private * @private
*/ */
async _searchWithoutFetch(searchCriteria) { async _searchWithoutFetch(searchCriteria) {
const imapUnderlying = this.connection.imap; const imapUnderlying = this.connection.imap
// If searching by UID and the list is too long, batch it
const UID_BATCH_SIZE = 500; return new Promise((resolve, reject) => {
// Detect UID search: ['UID', [array]] or ['UID', '1:1000'] imapUnderlying.search(searchCriteria, (err, uids) => {
if (Array.isArray(searchCriteria) && searchCriteria.length === 1 && Array.isArray(searchCriteria[0]) && searchCriteria[0][0] === 'UID' && Array.isArray(searchCriteria[0][1]) && searchCriteria[0][1].length > UID_BATCH_SIZE) { if (err) {
const allUids = searchCriteria[0][1]; reject(err)
let allResults = []; } else {
for (let i = 0; i < allUids.length; i += UID_BATCH_SIZE) { resolve(uids || [])
const batch = allUids.slice(i, i + UID_BATCH_SIZE); }
const batchCriteria = [ })
['UID', batch] })
];
// eslint-disable-next-line no-await-in-loop
const batchResult = await new Promise((resolve, reject) => {
imapUnderlying.search(batchCriteria, (err, uids) => {
if (err) reject(err);
else resolve(uids || []);
});
});
allResults = allResults.concat(batchResult);
}
return allResults;
} else {
return new Promise((resolve, reject) => {
imapUnderlying.search(searchCriteria, (err, uids) => {
if (err) {
reject(err);
} else {
resolve(uids || []);
}
});
});
}
} }
_createMailSummary(message) { _createMailSummary(message) {
@ -549,4 +487,4 @@ ImapService.EVENT_DELETED_MAIL = 'mailDeleted'
ImapService.EVENT_INITIAL_LOAD_DONE = 'initial load done' ImapService.EVENT_INITIAL_LOAD_DONE = 'initial load done'
ImapService.EVENT_ERROR = 'error' ImapService.EVENT_ERROR = 'error'
module.exports = ImapService module.exports = ImapService

59
domain/statistics-store.js
View file

@ -26,7 +26,6 @@ class StatisticsStore {
this.imapHash = this._computeImapHash(); this.imapHash = this._computeImapHash();
if (this.db) { if (this.db) {
this._autoMigrateRemoveIdCheck();
this._autoMigrateInstanceId(); this._autoMigrateInstanceId();
this._autoMigrateImapHash(); this._autoMigrateImapHash();
this._loadFromDatabase(); this._loadFromDatabase();
@ -34,48 +33,6 @@ class StatisticsStore {
debug('Statistics store initialized'); debug('Statistics store initialized');
} }
_autoMigrateRemoveIdCheck() {
debug('Starting CHECK(id = 1) migration check...');
// Remove CHECK(id = 1) constraint from statistics table if present
try {
const pragma = this.db.prepare("PRAGMA table_info(statistics)").all();
// Check if id column exists and if table has only one row (singleton)
const hasId = pragma.some(col => col.name === 'id');
if (hasId) {
// Check for CHECK constraint by inspecting sqlite_master
const tableSql = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='statistics'").get();
debug('Table SQL:', tableSql && tableSql.sql);
if (tableSql && tableSql.sql && tableSql.sql.includes('CHECK(id = 1)')) {
debug('Detected CHECK(id = 1) constraint on statistics table, migrating...');
// Backup data
const rows = this.db.prepare('SELECT * FROM statistics').all();
// Drop and recreate table without CHECK constraint
this.db.prepare('DROP TABLE IF EXISTS statistics').run();
this.db.prepare(`CREATE TABLE IF NOT EXISTS statistics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
instance_id TEXT NOT NULL,
largest_uid INTEGER NOT NULL DEFAULT 0,
hourly_data TEXT,
last_updated INTEGER NOT NULL,
imap_hash TEXT NULL
)`).run();
// Restore data
const insert = this.db.prepare('INSERT INTO statistics (id, instance_id, largest_uid, hourly_data, last_updated, imap_hash) VALUES (?, ?, ?, ?, ?, ?)');
rows.forEach(row => {
insert.run(row.id, row.instance_id, row.largest_uid, row.hourly_data, row.last_updated, row.imap_hash);
});
debug('Migration complete: CHECK(id = 1) removed from statistics table.');
} else {
debug('No CHECK(id = 1) constraint found in statistics table.');
}
} else {
debug('No id column found in statistics table.');
}
} catch (e) {
debug('Auto-migration for CHECK(id = 1) failed:', e.message);
}
}
_autoMigrateInstanceId() { _autoMigrateInstanceId() {
// Add and backfill instance_id for statistics table // Add and backfill instance_id for statistics table
try { try {
@ -156,9 +113,8 @@ class StatisticsStore {
debug(`Loaded from database: largestUid=${this.largestUid}, hourlyData=${this.hourlyData.length} entries`); debug(`Loaded from database: largestUid=${this.largestUid}, hourlyData=${this.hourlyData.length} entries`);
} else { } else {
// No row for this hash, insert new row // No row for this hash, insert new row
const instanceId = config.instanceId || this.imapHash; const insert = this.db.prepare('INSERT INTO statistics (imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?)');
const insert = this.db.prepare('INSERT OR REPLACE INTO statistics (instance_id, imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?, ?)'); insert.run(this.imapHash, 0, JSON.stringify([]), Date.now());
insert.run(instanceId, this.imapHash, 0, JSON.stringify([]), Date.now());
this.largestUid = 0; this.largestUid = 0;
this.hourlyData = []; this.hourlyData = [];
debug('Created new statistics row for imap_hash'); debug('Created new statistics row for imap_hash');
@ -171,18 +127,17 @@ class StatisticsStore {
_saveToDatabase() { _saveToDatabase() {
if (!this.db) return; if (!this.db) return;
try { try {
const instanceId = config.instanceId || this.imapHash;
const stmt = this.db.prepare(` const stmt = this.db.prepare(`
UPDATE statistics UPDATE statistics
SET largest_uid = ?, hourly_data = ?, last_updated = ? SET largest_uid = ?, hourly_data = ?, last_updated = ?
WHERE instance_id = ? AND imap_hash = ? WHERE imap_hash = ?
`); `);
const result = stmt.run(this.largestUid, JSON.stringify(this.hourlyData), Date.now(), instanceId, this.imapHash); const result = stmt.run(this.largestUid, JSON.stringify(this.hourlyData), Date.now(), this.imapHash);
// If no row was updated, insert new row // If no row was updated, insert new row
if (result.changes === 0) { if (result.changes === 0) {
const insert = this.db.prepare('INSERT INTO statistics (instance_id, imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?, ?)'); const insert = this.db.prepare('INSERT INTO statistics (imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?)');
insert.run(instanceId, this.imapHash, this.largestUid, JSON.stringify(this.hourlyData), Date.now()); insert.run(this.imapHash, this.largestUid, JSON.stringify(this.hourlyData), Date.now());
debug('Inserted new statistics row for instance_id and imap_hash'); debug('Inserted new statistics row for imap_hash');
} else { } else {
debug('Statistics saved to database'); debug('Statistics saved to database');
} }