I've combed over the past few months worth of list postings, and having
not seen the functionality in fetchmail that I needed, I have implemented
a patch for client-side seen tracking using the UID code present from the
pop3 implementation for imap.
I know there has been quite a bit of debate over whether client-side UID
is the way to go, however, in my particular instance in the way I wished
to use fetchmail it was the only way in which I could use it. Basically,
I needed the abliity to use fetchmail as "just another client" to read my
work imap mailbox and forward new messages to another address without
affecting the seen status on the parent mailbox (basically it should
sniff the mailbox for new mail and forward it to my wireless device
without affecting my other MUA's view of the mailbox). From what
I understand, the debate has more centered around whether fetchmail should
be used this way, but from my perspective it was the perfect tool to adapt
to do what I needed (isn't that the beauty of open source? :).
This patches imap_getrange to determine new messages based on a MD5 hash
of the Received and Message-ID headers using the UID code already present from
the pop3 UIDL implementation to determine the whether we've
seen the message already (functionally an implementation of
pop3_slowuidl, as there was no better way in the IMAP protocol which I saw
to implement it. I would be willing to implement a binary search similar
to Sunil's for the IMAP code if people found it useful. Based on the
situation in which I'm currently using this, I don't have any bandwidth
limitations or mailbox size limitations which would require the use of
such a technique). This also patches imap_mark_seen to mark the message
as seen through the UIDL code rather than a server side flag.
This code is dependent on both keep and uidl flags being set in the rc
file (it wouldn't be useful otherwise). I used the existing uidl flag
mainly because it applied and I don't understand lex and yacc enough to
modify the rcfile code :). This also does not update any documentation
(for instance the man page says uidl applies only to pop3).
This is the first time I've contributed code to an open source project,
and the first time in a while I've coded in C. Please let me know if it
sucks or not. If anyone has a better way I should have implemented this,
please let me know and I'll go back and rewrite it in such a way as to
find it's way into the mainstream version (assuming of course the overall
debate as to whether fetchmail should be used this way is solved).
Patch follows. Thanks for giving me the framework to do what I needed to,
fetchmail is truly an outstanding and adaptable program.
Clint Sharp
--- fetchmail-6.2.5.orig/imap.c 2003-10-15 12:17:41.000000000 -0700
+++ fetchmail-6.2.5/imap.c 2004-03-21 18:56:22.000000000 -0800
@@ -717,7 +717,126 @@
startcount = 1;
/* OK, now get a count of unseen messages and their indices */
- if (!ctl->fetchall && count > 0)
+
+ /* Adding support for using Unique IDs (although we have to generate
+ * them for ourselves) to determine which messages we should fetch.
+ * We will rely on the uidl parameter already in the config file
+ * to specify this parameter for us
+ */
+ if (ctl->server.uidl && ctl->keep)
+ {
+ if (unseen_messages)
+ free(unseen_messages);
+ unseen_messages = xmalloc(count * sizeof(unsigned int));
+ memset(unseen_messages, 0, count * sizeof(unsigned int));
+ unseen = 0;
+
+ gen_send(sock, "FETCH 1:* (BODY.PEEK[HEADER.FIELDS (MESSAGE-ID
RECEIVED)])");
+ do
+ {
+ char *ep;
+ ok = gen_recv(sock, buf, sizeof(buf));
+ if (ok != 0)
+ {
+ report(stderr, GT_("retreiving headers for IMAP UIDL hash
failed\n"));
+ return(PS_PROTOCOL);
+ }
+ else
+ {
+ unsigned int um, size;
+
+ if (sscanf(buf, "* %u FETCH %*s %*s %*s {%u}", &um, &size) == 2)
+ {
+
+ char *header = NULL;
+
+ /* Read all subsequent lines until the line begins with ).
*/
+ ok = 0;
+ do
+ {
+ ok = gen_recv(sock, buf, sizeof(buf));
+ if (ok != 0)
+ {
+ report(stderr, "reading headers for IMAP UIDL hash
failed\n");
+ return(PS_PROTOCOL);
+ }
+ if (header == NULL)
+ {
+ header = xmalloc(strlen(buf)+3);
+ header[0] = '\0';
+ }
+ else
+ {
+ header = xrealloc(header,
strlen(header)+strlen(buf)+3);
+ }
+ if (strlen(buf)>0 && buf[0] != ')')
+ {
+ int hl;
+ strcpy(header+strlen(header), buf);
+ hl = strlen(header);
+ header[strlen(header)] = '\r';
+ header[hl] = '\n';
+ header[hl+1] = '\0';
+ }
+ } while (buf[0] != ')' && ok == 0);
+
+ /* We now have the header so lets make a hash and
determine if we've seen it before */
+ char headdigest[33];
+ strcpy(headdigest, MD5Digest((unsigned char *)header));
+
+ if (outlevel >= O_DEBUG)
+ report(stderr, "header hash: %s\n", headdigest);
+
+ if (header != NULL)
+ free(header);
+
+ /* The following code was ripped almost verbatim from
pop3.c in pop3_getrange */
+
+ struct idlist *old, *new;
+
+ new = save_str(&ctl->newsaved, headdigest, UID_UNSEEN);
+ new->val.status.num = um;
+
+ if ((old = str_in_list(&ctl->oldsaved, headdigest, FALSE)))
+ {
+ flag mark = old->val.status.mark;
+ if (mark == UID_DELETED || mark == UID_EXPUNGED)
+ {
+ if (outlevel >= O_VERBOSE)
+ report(stderr, GT_("id=%s (num=%d) was deleted
but is still present!\n"), headdigest, um);
+ /* mark it as seen now */
+ old->val.status.mark = mark = UID_SEEN;
+ }
+ new->val.status.mark = mark;
+ if (mark == UID_UNSEEN && um <= count)
+ {
+ unseen_messages[unseen++] = um;
+ if (outlevel >= O_DEBUG)
+ report(stderr, GT_("%u is unseen.\n"), um);
+ if (startcount > um)
+ startcount = um;
+ }
+ }
+ else if (um <= count)
+ {
+ unseen_messages[unseen++] = um;
+ if (outlevel >= O_DEBUG)
+ report(stderr, GT_("%u is unseen.\n"), um);
+ if (startcount > um)
+ startcount = um;
+ /* add it to oldsaved also! In case, we do not
+ * swap the lists (say, due to socket error),
+ * the same mail will not be downloaded again.
+ */
+ old = save_str(&ctl->oldsaved, headdigest, UID_UNSEEN);
+ old->val.status.num = um;
+ }
+ }
+ }
+ } while
+ (tag[0] != '\0' && strncmp(buf, tag, strlen(tag)));
+ }
+ else if (!ctl->fetchall && count > 0)
{
if (unseen_messages)
free(unseen_messages);
@@ -1087,11 +1206,34 @@
static int imap_mark_seen(int sock, struct query *ctl, int number)
/* mark the given message as seen */
{
+ if (ctl->server.uidl && ctl->keep)
+ {
+ struct idlist *sdp;
+
+ /* Code again ripped from pop.c */
+ if ((sdp = id_find(&ctl->newsaved, number)))
+ sdp->val.status.mark = UID_SEEN;
+ else
+ return(PS_ERROR);
+ /* mark it as seen in oldsaved also! In case, we do not swap the lists
+ * (say, due to socket error), the same mail will not be downloaded
+ * again.
+ */
+ if ((sdp = id_find(&ctl->oldsaved, number)))
+ sdp->val.status.mark = UID_SEEN;
+ else
+ return(PS_ERROR);
+
+ return(PS_SUCCESS);
+ }
+ else
+ {
return(gen_transact(sock,
imap_version == IMAP4
? "STORE %d +FLAGS.SILENT (\\Seen)"
: "STORE %d +FLAGS (\\Seen)",
number));
+ }
}
static int imap_logout(int sock, struct query *ctl)