jmb/48hr.email
1
0
Fork 0
mirror of https://github.com/Crazyco-xyz/48hr.email.git synced 2026-02-14 17:19:35 +01:00
Code Issues Projects Releases Packages Wiki Activity

Compare commits

base: jmb:51d608c2c00b39e939fa7d51c0d12976db42756e
Branches Tags
jmb:main
..
compare: jmb:f8ce98a5846567298e800dcb532c73224fb08e06
Branches Tags
jmb:main

No commits in common. "51d608c2c00b39e939fa7d51c0d12976db42756e" and "f8ce98a5846567298e800dcb532c73224fb08e06" have entirely different histories.
51d608c2c0 ... f8ce98a584

2 changed files with 39 additions and 146 deletions
Show stats Expand all files Collapse all files

106
application/imap-service.js
View file

@ -253,46 +253,17 @@ class ImapService extends EventEmitter {
// IMAP date filters are unreliable - some servers search internal date, not Date header
// Always fetch all UIDs and filter by date header in JavaScript instead
const deleteOlderThan = helper.purgeTimeStamp();
const searchQuery = [
['!DELETED']
];
uids = await this._searchWithoutFetch(searchQuery);
if (uids.length === 0) return;
debug(`Starting deleteOldMails. Total UIDs: ${uids.length}`);
const HEADER_BATCH_SIZE = this.config.imap.fetchChunkSize
const concurrency = this.config.imap.fetchConcurrency || 4;
// Chunk UIDs
const uidChunks = [];
for (let i = 0; i < uids.length; i += HEADER_BATCH_SIZE) {
uidChunks.push(uids.slice(i, i + HEADER_BATCH_SIZE));
}
let headers = [];
let workerId = 0;
const runNext = async() => {
if (workerId >= uidChunks.length) return;
const chunkId = workerId++;
const batch = uidChunks[chunkId];
try {
const batchHeaders = await Promise.race([
this._getMailHeaders(batch),
new Promise((_, reject) => setTimeout(() => reject(new Error('Header fetch timeout')), 30000))
]);
headers = headers.concat(batchHeaders);
} catch (err) {
debug(`ERROR in batch ${chunkId+1}: ${err.message}`);
}
await runNext();
};
// Start workers
const pool = [];
const workers = Math.min(concurrency, uidChunks.length);
for (let i = 0; i < workers; i++) {
pool.push(runNext());
}
await Promise.all(pool);
const deleteOlderThan = helper.purgeTimeStamp();
const exampleUids = this.config.email.examples.uids.map(x => parseInt(x));
const headers = await this._getMailHeaders(uids);
// Get locked inboxes if available
let lockedAddresses = [];
@ -305,26 +276,19 @@ class ImapService extends EventEmitter {
}
}
// Ensure exampleUids is defined before filtering
const exampleUids = this.config.email.examples && this.config.email.examples.uids ?
this.config.email.examples.uids.map(x => parseInt(x)) : [];
// Filter out mails that are too new, whitelisted, or belong to locked inboxes
let filtered = headers;
// Exclude exampleUids
filtered = filtered.filter(mail => !exampleUids.includes(parseInt(mail.attributes.uid)));
// Exclude locked inboxes
filtered = filtered.filter(mail => {
const toDelete = headers
.filter(mail => {
const date = mail.attributes.date;
const uid = parseInt(mail.attributes.uid);
const toAddresses = Array.isArray(mail.parts[0].body.to) ?
mail.parts[0].body.to.map(a => a.toLowerCase()) : [String(mail.parts[0].body.to).toLowerCase()];
return !toAddresses.some(addr => lockedAddresses.includes(addr));
});
// Exclude too new (compare timestamps for Date objects)
filtered = filtered.filter(mail => mail.attributes.date && mail.attributes.date.getTime() <= deleteOlderThan.getTime());
const toDelete = filtered.map(mail => parseInt(mail.attributes.uid));
if (exampleUids.includes(uid)) return false;
if (toAddresses.some(addr => lockedAddresses.includes(addr))) return false;
return date <= deleteOlderThan;
})
.map(mail => parseInt(mail.attributes.uid));
if (toDelete.length === 0) {
debug('No mails to delete. (after locked inbox exclusion)');
@ -332,16 +296,12 @@ class ImapService extends EventEmitter {
}
debug(`Deleting mails ${toDelete}`);
// Batch deletes to avoid IMAP argument limits
const BATCH_SIZE = this.config.imap.fetchChunkSize;
for (let i = 0; i < toDelete.length; i += BATCH_SIZE) {
const batch = toDelete.slice(i, i + BATCH_SIZE);
await this.connection.deleteMessage(batch);
batch.forEach(uid => {
await this.connection.deleteMessage(toDelete);
toDelete.forEach(uid => {
this.emit(ImapService.EVENT_DELETED_MAIL, uid);
});
}
}
/**
@ -378,39 +338,17 @@ class ImapService extends EventEmitter {
* @private
*/
async _searchWithoutFetch(searchCriteria) {
const imapUnderlying = this.connection.imap;
// If searching by UID and the list is too long, batch it
const UID_BATCH_SIZE = 500;
// Detect UID search: ['UID', [array]] or ['UID', '1:1000']
if (Array.isArray(searchCriteria) && searchCriteria.length === 1 && Array.isArray(searchCriteria[0]) && searchCriteria[0][0] === 'UID' && Array.isArray(searchCriteria[0][1]) && searchCriteria[0][1].length > UID_BATCH_SIZE) {
const allUids = searchCriteria[0][1];
let allResults = [];
for (let i = 0; i < allUids.length; i += UID_BATCH_SIZE) {
const batch = allUids.slice(i, i + UID_BATCH_SIZE);
const batchCriteria = [
['UID', batch]
];
// eslint-disable-next-line no-await-in-loop
const batchResult = await new Promise((resolve, reject) => {
imapUnderlying.search(batchCriteria, (err, uids) => {
if (err) reject(err);
else resolve(uids || []);
});
});
allResults = allResults.concat(batchResult);
}
return allResults;
} else {
const imapUnderlying = this.connection.imap
return new Promise((resolve, reject) => {
imapUnderlying.search(searchCriteria, (err, uids) => {
if (err) {
reject(err);
reject(err)
} else {
resolve(uids || []);
}
});
});
resolve(uids || [])
}
})
})
}
_createMailSummary(message) {

59
domain/statistics-store.js
View file

@ -26,7 +26,6 @@ class StatisticsStore {
this.imapHash = this._computeImapHash();
if (this.db) {
this._autoMigrateRemoveIdCheck();
this._autoMigrateInstanceId();
this._autoMigrateImapHash();
this._loadFromDatabase();
@ -34,48 +33,6 @@ class StatisticsStore {
debug('Statistics store initialized');
}
_autoMigrateRemoveIdCheck() {
debug('Starting CHECK(id = 1) migration check...');
// Remove CHECK(id = 1) constraint from statistics table if present
try {
const pragma = this.db.prepare("PRAGMA table_info(statistics)").all();
// Check if id column exists and if table has only one row (singleton)
const hasId = pragma.some(col => col.name === 'id');
if (hasId) {
// Check for CHECK constraint by inspecting sqlite_master
const tableSql = this.db.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='statistics'").get();
debug('Table SQL:', tableSql && tableSql.sql);
if (tableSql && tableSql.sql && tableSql.sql.includes('CHECK(id = 1)')) {
debug('Detected CHECK(id = 1) constraint on statistics table, migrating...');
// Backup data
const rows = this.db.prepare('SELECT * FROM statistics').all();
// Drop and recreate table without CHECK constraint
this.db.prepare('DROP TABLE IF EXISTS statistics').run();
this.db.prepare(`CREATE TABLE IF NOT EXISTS statistics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
instance_id TEXT NOT NULL,
largest_uid INTEGER NOT NULL DEFAULT 0,
hourly_data TEXT,
last_updated INTEGER NOT NULL,
imap_hash TEXT NULL
)`).run();
// Restore data
const insert = this.db.prepare('INSERT INTO statistics (id, instance_id, largest_uid, hourly_data, last_updated, imap_hash) VALUES (?, ?, ?, ?, ?, ?)');
rows.forEach(row => {
insert.run(row.id, row.instance_id, row.largest_uid, row.hourly_data, row.last_updated, row.imap_hash);
});
debug('Migration complete: CHECK(id = 1) removed from statistics table.');
} else {
debug('No CHECK(id = 1) constraint found in statistics table.');
}
} else {
debug('No id column found in statistics table.');
}
} catch (e) {
debug('Auto-migration for CHECK(id = 1) failed:', e.message);
}
}
_autoMigrateInstanceId() {
// Add and backfill instance_id for statistics table
try {
@ -156,9 +113,8 @@ class StatisticsStore {
debug(`Loaded from database: largestUid=${this.largestUid}, hourlyData=${this.hourlyData.length} entries`);
} else {
// No row for this hash, insert new row
const instanceId = config.instanceId || this.imapHash;
const insert = this.db.prepare('INSERT OR REPLACE INTO statistics (instance_id, imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?, ?)');
insert.run(instanceId, this.imapHash, 0, JSON.stringify([]), Date.now());
const insert = this.db.prepare('INSERT INTO statistics (imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?)');
insert.run(this.imapHash, 0, JSON.stringify([]), Date.now());
this.largestUid = 0;
this.hourlyData = [];
debug('Created new statistics row for imap_hash');
@ -171,18 +127,17 @@ class StatisticsStore {
_saveToDatabase() {
if (!this.db) return;
try {
const instanceId = config.instanceId || this.imapHash;
const stmt = this.db.prepare(`
UPDATE statistics
SET largest_uid = ?, hourly_data = ?, last_updated = ?
WHERE instance_id = ? AND imap_hash = ?
WHERE imap_hash = ?
`);
const result = stmt.run(this.largestUid, JSON.stringify(this.hourlyData), Date.now(), instanceId, this.imapHash);
const result = stmt.run(this.largestUid, JSON.stringify(this.hourlyData), Date.now(), this.imapHash);
// If no row was updated, insert new row
if (result.changes === 0) {
const insert = this.db.prepare('INSERT INTO statistics (instance_id, imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?, ?)');
insert.run(instanceId, this.imapHash, this.largestUid, JSON.stringify(this.hourlyData), Date.now());
debug('Inserted new statistics row for instance_id and imap_hash');
const insert = this.db.prepare('INSERT INTO statistics (imap_hash, largest_uid, hourly_data, last_updated) VALUES (?, ?, ?, ?)');
insert.run(this.imapHash, this.largestUid, JSON.stringify(this.hourlyData), Date.now());
debug('Inserted new statistics row for imap_hash');
} else {
debug('Statistics saved to database');
}