[Fix]: Chunk everything

I hate IMAP, respectfully.

application/imap-service.js

@@ -253,17 +253,46 @@ class ImapService extends EventEmitter {
 
         // IMAP date filters are unreliable - some servers search internal date, not Date header
         // Always fetch all UIDs and filter by date header in JavaScript instead
+        const deleteOlderThan = helper.purgeTimeStamp();
         const searchQuery = [
             ['!DELETED']
         ];
-
         uids = await this._searchWithoutFetch(searchQuery);
 
         if (uids.length === 0) return;
 
-        const deleteOlderThan = helper.purgeTimeStamp();
+        debug(`Starting deleteOldMails. Total UIDs: ${uids.length}`);
-        const exampleUids = this.config.email.examples.uids.map(x => parseInt(x));
+        const HEADER_BATCH_SIZE = this.config.imap.fetchChunkSize
-        const headers = await this._getMailHeaders(uids);
+        const concurrency = this.config.imap.fetchConcurrency || 4;
+        // Chunk UIDs
+        const uidChunks = [];
+        for (let i = 0; i < uids.length; i += HEADER_BATCH_SIZE) {
+            uidChunks.push(uids.slice(i, i + HEADER_BATCH_SIZE));
+        }
+        let headers = [];
+        let workerId = 0;
+        const runNext = async() => {
+            if (workerId >= uidChunks.length) return;
+            const chunkId = workerId++;
+            const batch = uidChunks[chunkId];
+            try {
+                const batchHeaders = await Promise.race([
+                    this._getMailHeaders(batch),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error('Header fetch timeout')), 30000))
+                ]);
+                headers = headers.concat(batchHeaders);
+            } catch (err) {
+                debug(`ERROR in batch ${chunkId+1}: ${err.message}`);
+            }
+            await runNext();
+        };
+        // Start workers
+        const pool = [];
+        const workers = Math.min(concurrency, uidChunks.length);
+        for (let i = 0; i < workers; i++) {
+            pool.push(runNext());
+        }
+        await Promise.all(pool);
 
         // Get locked inboxes if available
         let lockedAddresses = [];
@@ -276,19 +305,26 @@ class ImapService extends EventEmitter {
             }
         }
 
-        // Filter out mails that are too new, whitelisted, or belong to locked inboxes
+        // Ensure exampleUids is defined before filtering
-        const toDelete = headers
+        const exampleUids = this.config.email.examples && this.config.email.examples.uids ?
-            .filter(mail => {
+            this.config.email.examples.uids.map(x => parseInt(x)) : [];
-                const date = mail.attributes.date;
-                const uid = parseInt(mail.attributes.uid);
-                const toAddresses = Array.isArray(mail.parts[0].body.to) ?
-                    mail.parts[0].body.to.map(a => a.toLowerCase()) : [String(mail.parts[0].body.to).toLowerCase()];
 
-                if (exampleUids.includes(uid)) return false;
+        // Filter out mails that are too new, whitelisted, or belong to locked inboxes
-                if (toAddresses.some(addr => lockedAddresses.includes(addr))) return false;
+        let filtered = headers;
-                return date <= deleteOlderThan;
+
-            })
+        // Exclude exampleUids
-            .map(mail => parseInt(mail.attributes.uid));
+        filtered = filtered.filter(mail => !exampleUids.includes(parseInt(mail.attributes.uid)));
+
+        // Exclude locked inboxes
+        filtered = filtered.filter(mail => {
+            const toAddresses = Array.isArray(mail.parts[0].body.to) ?
+                mail.parts[0].body.to.map(a => a.toLowerCase()) : [String(mail.parts[0].body.to).toLowerCase()];
+            return !toAddresses.some(addr => lockedAddresses.includes(addr));
+        });
+
+        // Exclude too new (compare timestamps for Date objects)
+        filtered = filtered.filter(mail => mail.attributes.date && mail.attributes.date.getTime() <= deleteOlderThan.getTime());
+        const toDelete = filtered.map(mail => parseInt(mail.attributes.uid));
 
         if (toDelete.length === 0) {
             debug('No mails to delete. (after locked inbox exclusion)');
@@ -297,7 +333,7 @@ class ImapService extends EventEmitter {
 
         debug(`Deleting mails ${toDelete}`);
         // Batch deletes to avoid IMAP argument limits
-        const BATCH_SIZE = 100;
+        const BATCH_SIZE = this.config.imap.fetchChunkSize;
         for (let i = 0; i < toDelete.length; i += BATCH_SIZE) {
             const batch = toDelete.slice(i, i + BATCH_SIZE);
             await this.connection.deleteMessage(batch);
@@ -342,17 +378,39 @@ class ImapService extends EventEmitter {
      * @private
      */
     async _searchWithoutFetch(searchCriteria) {
-        const imapUnderlying = this.connection.imap
+        const imapUnderlying = this.connection.imap;
-
+        // If searching by UID and the list is too long, batch it
-        return new Promise((resolve, reject) => {
+        const UID_BATCH_SIZE = 500;
-            imapUnderlying.search(searchCriteria, (err, uids) => {
+        // Detect UID search: ['UID', [array]] or ['UID', '1:1000']
-                if (err) {
+        if (Array.isArray(searchCriteria) && searchCriteria.length === 1 && Array.isArray(searchCriteria[0]) && searchCriteria[0][0] === 'UID' && Array.isArray(searchCriteria[0][1]) && searchCriteria[0][1].length > UID_BATCH_SIZE) {
-                    reject(err)
+            const allUids = searchCriteria[0][1];
-                } else {
+            let allResults = [];
-                    resolve(uids || [])
+            for (let i = 0; i < allUids.length; i += UID_BATCH_SIZE) {
-                }
+                const batch = allUids.slice(i, i + UID_BATCH_SIZE);
-            })
+                const batchCriteria = [
-        })
+                    ['UID', batch]
+                ];
+                // eslint-disable-next-line no-await-in-loop
+                const batchResult = await new Promise((resolve, reject) => {
+                    imapUnderlying.search(batchCriteria, (err, uids) => {
+                        if (err) reject(err);
+                        else resolve(uids || []);
+                    });
+                });
+                allResults = allResults.concat(batchResult);
+            }
+            return allResults;
+        } else {
+            return new Promise((resolve, reject) => {
+                imapUnderlying.search(searchCriteria, (err, uids) => {
+                    if (err) {
+                        reject(err);
+                    } else {
+                        resolve(uids || []);
+                    }
+                });
+            });
+        }
     }
 
     _createMailSummary(message) {
@@ -491,4 +549,4 @@ ImapService.EVENT_DELETED_MAIL = 'mailDeleted'
 ImapService.EVENT_INITIAL_LOAD_DONE = 'initial load done'
 ImapService.EVENT_ERROR = 'error'
 
-module.exports = ImapService
+module.exports = ImapService