48hr.email/application/imap-service.js

366 lines
10 KiB
JavaScript
Raw Normal View History

2023-11-01 11:48:19 +01:00
const EventEmitter = require('events')
const imaps = require('imap-simple')
const {simpleParser} = require('mailparser')
const addressparser = require('nodemailer/lib/addressparser')
const pSeries = require('p-series')
const retry = require('async-retry')
const debug = require('debug')('48hr-email:imap')
const _ = require('lodash')
const moment = require('moment')
2023-11-01 11:48:19 +01:00
const Mail = require('../domain/mail')
const Helper = require('./helper')
const helper = new(Helper)
2023-11-01 11:48:19 +01:00
// Just adding some missing functions to imap-simple... :-)
/**
* Deletes the specified message(s).
*
* @param {string|Array} uid The uid or array of uids indicating the messages to be deleted
* @param {function} [callback] Optional callback, receiving signature (err)
* @returns {undefined|Promise} Returns a promise when no callback is specified, resolving when the action succeeds.
* @memberof ImapSimple
*/
imaps.ImapSimple.prototype.deleteMessage = function (uid, callback) {
var self = this;
if (callback) {
return nodeify(self.deleteMessage(uid), callback);
}
return new Promise(function (resolve, reject) {
self.imap.addFlags(uid, '\\Deleted', function (err) {
if (err) {
reject(err);
return;
}
self.imap.expunge( function (err) {
if (err) {
reject(err);
return;
}
resolve();
});
});
});
};
/**
* Close a mailbox
*
2023-11-02 07:41:52 +01:00
* @param {boolean} [autoExpunge=true] If autoExpunge is true, any messages marked as Deleted in the currently open mailbox will be removed
2023-11-01 11:48:19 +01:00
* @param {function} [callback] Optional callback, receiving signature (err)
* @returns {undefined|Promise} Returns a promise when no callback is specified, resolving to `boxName`
* @memberof ImapSimple
*/
imaps.ImapSimple.prototype.closeBox = function (autoExpunge=true, callback) {
var self = this;
if (typeof(autoExpunge) == 'function'){
callback = autoExpunge;
autoExpunge = true;
}
if (callback) {
return nodeify(this.closeBox(autoExpunge), callback);
}
return new Promise(function (resolve, reject) {
self.imap.closeBox(autoExpunge, function (err, result) {
if (err) {
reject(err);
return;
}
resolve(result);
});
});
};
/**
* Fetches emails from the imap server. It is a facade against the more complicated imap-simple api. It keeps the connection
* as a member field.
*
* With this abstraction it would be easy to replace this with any inbound mail service like mailgun.com.
*/
class ImapService extends EventEmitter {
constructor(config) {
super()
this.config = config
/**
* Set of emitted UIDs. Listeners should get each email only once.
* @type {Set<any>}
*/
this.loadedUids = new Set()
this.connection = null
this.initialLoadDone = false
}
async connectAndLoadMessages() {
const configWithListener = {
...this.config,
// 'onmail' adds a callback when new mails arrive. With this we can keep the imap refresh interval very low (or even disable it).
onmail: () => this._doOnNewMail()
}
this.once(ImapService.EVENT_INITIAL_LOAD_DONE, () =>
this._doAfterInitialLoad()
)
await this._connectWithRetry(configWithListener)
// Load all messages in the background. (ASYNC)
this._loadMailSummariesAndEmitAsEvents()
}
async _connectWithRetry(configWithListener) {
try {
await retry(
async _bail => {
// If anything throws, we retry
this.connection = await imaps.connect(configWithListener)
this.connection.on('error', err => {
// We assume that the app will be restarted after a crash.
console.error(
'got fatal error during imap operation, stop app.',
err
)
this.emit('error', err)
})
await this.connection.openBox('INBOX')
debug('connected to imap')
},
{
retries: 5
}
)
} catch (error) {
console.error('can not connect, even with retry, stop app', error)
throw error
}
}
_doOnNewMail() {
// Only react to new mails after the initial load, otherwise it might load the same mails twice.
if (this.initialLoadDone) {
this._loadMailSummariesAndEmitAsEvents()
}
}
_doAfterInitialLoad() {
// During initial load we ignored new incoming emails. In order to catch up with those, we have to refresh
// the mails once after the initial load. (async)
this._loadMailSummariesAndEmitAsEvents()
// If the above trigger on new mails does not work reliable, we have to regularly check
// for new mails on the server. This is done only after all the mails have been loaded for the
// first time. (Note: set the refresh higher than the time it takes to download the mails).
if (this.config.imap.refreshIntervalSeconds) {
setInterval(
() => this._loadMailSummariesAndEmitAsEvents(),
this.config.imap.refreshIntervalSeconds * 1000
)
}
}
async _loadMailSummariesAndEmitAsEvents() {
// UID: Unique id of a message.
const uids = await this._getAllUids()
const newUids = uids.filter(uid => !this.loadedUids.has(uid))
// Optimize by fetching several messages (but not all) with one 'search' call.
// fetching all at once might be more efficient, but then it takes long until we see any messages
// in the frontend. With a small chunk size we ensure that we see the newest emails after a few seconds after
// restart.
const uidChunks = _.chunk(newUids, 20)
// Creates an array of functions. We do not start the search now, we just create the function.
const fetchFunctions = uidChunks.map(uidChunk => () =>
this._getMailHeadersAndEmitAsEvents(uidChunk)
)
await pSeries(fetchFunctions)
if (!this.initialLoadDone) {
this.initialLoadDone = true
this.emit(ImapService.EVENT_INITIAL_LOAD_DONE)
}
}
/**
*
* @param {Date} deleteMailsBefore delete mails before this date instance
*/
async deleteOldMails(deleteMailsBefore) {
let uids = []
if (helper.moreThanOneDay(moment(), deleteMailsBefore)) {
uids = await this._searchWithoutFetch([
['!DELETED'],
['BEFORE', deleteMailsBefore]
])
} else {
2024-11-11 19:15:19 +01:00
//fetch mails from date -1day (calculated in MS) to avoid wasting resources
deleteMailsBefore = new Date(moment() - 24 * 60 * 60 * 1000)
uids = await this._searchWithoutFetch([
2024-11-11 19:15:19 +01:00
['!DELETED'],
['BEFORE', deleteMailsBefore]
])
}
2023-11-01 11:48:19 +01:00
if (uids.length === 0) {
return
}
const DeleteOlderThan = helper.purgeTimeStamp()
const uidsWithHeaders = await this._getMailHeaders(uids)
uidsWithHeaders.forEach(mail => {
2024-10-21 01:52:33 +02:00
if (mail['attributes'].date > DeleteOlderThan || this.config.email.examples.uids.includes(parseInt(mail['attributes'].uid))) {
uids = uids.filter(uid => uid !== mail['attributes'].uid)
}
})
if (uids.length === 0) {
debug('no mails to delete.')
return
}
debug(`deleting mails ${uids}`)
await this.connection.deleteMessage(uids)
2024-10-02 00:05:48 +02:00
uids.forEach(uid => this.emit(ImapService.EVENT_DELETED_MAIL, uid))
console.log(`deleted ${uids.length} old messages.`)
2023-11-01 11:48:19 +01:00
}
2023-11-02 06:25:22 +01:00
/**
*
* @param uid delete specific mail per UID
*/
async deleteSpecificEmail(uid) {
debug(`deleting mails ${uid}`)
if (!this.config.email.examples.uids.includes(parseInt(uid))) {
2024-10-02 02:36:25 +02:00
await this.connection.deleteMessage(uid)
console.log(`deleted mail with UID: ${uid}.`)
this.emit(ImapService.EVENT_DELETED_MAIL, uid)
}
2023-11-02 06:25:22 +01:00
}
2023-11-01 11:48:19 +01:00
/**
* Helper method because ImapSimple#search also fetches each message. We just need the uids here.
*
* @param {Object} searchCriteria (see ImapSimple#search)
* @returns {Promise<Array<Int>>} Array of UIDs
* @private
*/
async _searchWithoutFetch(searchCriteria) {
const imapUnderlying = this.connection.imap
return new Promise((resolve, reject) => {
imapUnderlying.search(searchCriteria, (err, uids) => {
if (err) {
reject(err)
} else {
resolve(uids || [])
}
})
})
}
_createMailSummary(message) {
const headerPart = message.parts[0].body
const to = headerPart.to
.flatMap(to => addressparser(to))
// The address also contains the name, just keep the email
.map(addressObj => addressObj.address)
const from = headerPart.from.flatMap(from => addressparser(from))
2024-09-29 10:20:20 +02:00
// Specify default subject, in case none exists.
2024-09-28 09:04:05 +02:00
let subject = "No Subject"
try {
subject = headerPart.subject[0]
} catch {
// Do nothing
}
2023-11-01 11:48:19 +01:00
const date = headerPart.date[0]
const {uid} = message.attributes
return Mail.create(to, from, date, subject, uid)
}
2024-09-28 04:44:13 +02:00
async fetchOneFullMail(to, uid, raw = false) {
2023-11-01 11:48:19 +01:00
if (!this.connection) {
// Here we 'fail fast' instead of waiting for the connection.
throw new Error('imap connection not ready')
}
debug(`fetching full message ${uid}`)
// For security we also filter TO, so it is harder to just enumerate all messages.
const searchCriteria = [['UID', uid], ['TO', to]]
const fetchOptions = {
bodies: ['HEADER', ''], // Empty string means full body
markSeen: false
}
const messages = await this.connection.search(searchCriteria, fetchOptions)
if (messages.length === 0) {
2024-09-28 05:21:51 +02:00
return false
} else if (!raw) {
2024-09-28 04:44:13 +02:00
const fullBody = await _.find(messages[0].parts, {which: ''})
return simpleParser(fullBody.body)
} else {
return messages[0].parts[1].body
}
2023-11-01 11:48:19 +01:00
}
2024-09-28 04:44:13 +02:00
2023-11-01 11:48:19 +01:00
async _getAllUids() {
// We ignore mails that are flagged as DELETED, but have not been removed (expunged) yet.
const uids = await this._searchWithoutFetch([['!DELETED']])
// Create copy to not mutate the original array. Sort with newest first (DESC).
return [...uids].sort().reverse()
}
async _getMailHeadersAndEmitAsEvents(uids) {
try {
const mails = await this._getMailHeaders(uids)
mails.forEach(mail => {
this.loadedUids.add(mail.attributes.uid)
// Some broadcast messages have no TO field. We have to ignore those messages.
if (mail.parts[0].body.to) {
this.emit(ImapService.EVENT_NEW_MAIL, this._createMailSummary(mail))
}
})
} catch (error) {
debug('can not fetch', error)
throw error
}
}
async _getMailHeaders(uids) {
const fetchOptions = {
envelope: true,
2023-11-01 11:48:19 +01:00
bodies: ['HEADER.FIELDS (FROM TO SUBJECT DATE)'],
struct: false
}
const searchCriteria = [['UID', ...uids]]
return this.connection.search(searchCriteria, fetchOptions)
}
}
// Consumers should use these constants:
ImapService.EVENT_NEW_MAIL = 'mail'
ImapService.EVENT_DELETED_MAIL = 'mailDeleted'
ImapService.EVENT_INITIAL_LOAD_DONE = 'initial load done'
ImapService.EVENT_ERROR = 'error'
module.exports = ImapService