interimap: fix handling of mod-sequence values greater or equal than 2 << 63.

SQLite processes every INTEGER values as a 8-byte signed integer, so we
need to manually do the conversion from/to uint64_t client-side if we
don't want to overflow or receive floats.

https://www.sqlite.org/datatype3.html#storage_classes_and_datatypes
http://jakegoulding.com/blog/2011/02/06/sqlite-64-bit-integers/

We could also do the same trick for local/remote UIDs, UIDVALITY and
UIDNEXT values to slim the database down at the expense of pre/post-
processing.  (Values of SQLite's INTEGER class are 1, 2, 3, 4, 6, or 8
bytes signed integers depending on the manitudes, so we could save some
space for values ≥2³¹.)  But that seems a little overkill.

diff --git a/Changelog b/Changelog
index 3d8cd72a916d0b6f3525be8f0344be5274b44c10..251d5dc08fdf66f513028e83627f4df181fa023d 100644 (file)
--- a/Changelog
+++ b/Changelog
@@ -40,6 +40,8 @@ interimap (0.5) upstream;
    RFC 3501 sec. 6.3.4).
  - interimap: SQLite were not enforcing foreign key constraints (setting
    the 'foreign_keys' PRAGMA during a transaction is a documented no-op).
+ - interimap: fix handling of mod-sequence values greater or equal than
+   2 << 63.
 
  -- Guilhem Moulin <guilhem@fripost.org>  Fri, 10 May 2019 00:58:14 +0200
 

diff --git a/interimap b/interimap
index 78f50fa046e4160d969cd061048df7624e2219e9..2dd0eb53701696dd453cfdfb934f3c04efbb9723 100755 (executable)
--- a/interimap
+++ b/interimap
@@ -352,7 +352,7 @@ fail(undef, "Local and remote namespaces are neither both flat nor both hierarch
             # no UNIQUE constraint on UIDVALIDITY as two mailboxes may share the same value
             q{UIDVALIDITY   UNSIGNED INT    NOT NULL CHECK (UIDVALIDITY > 0)},
             q{UIDNEXT       UNSIGNED INT    NOT NULL}, # 0 initially
-            q{HIGHESTMODSEQ UNSIGNED BIGINT NOT NULL}  # 0 initially
+            q{HIGHESTMODSEQ UNSIGNED BIGINT NOT NULL}  # 0 initially (/!\ converted to 8-byte signed integer)
             # one-to-one correspondence between local.idx and remote.idx
         ],
         remote => [
@@ -360,7 +360,7 @@ fail(undef, "Local and remote namespaces are neither both flat nor both hierarch
             # no UNIQUE constraint on UIDVALIDITY as two mailboxes may share the same value
             q{UIDVALIDITY   UNSIGNED INT    NOT NULL CHECK (UIDVALIDITY > 0)},
             q{UIDNEXT       UNSIGNED INT    NOT NULL}, # 0 initially
-            q{HIGHESTMODSEQ UNSIGNED BIGINT NOT NULL}  # 0 initially
+            q{HIGHESTMODSEQ UNSIGNED BIGINT NOT NULL}  # 0 initially (/!\ converted to 8-byte signed integer)
             # one-to-one correspondence between local.idx and remote.idx
         ],
         mapping => [
@@ -713,18 +713,6 @@ my $ATTRS = join ' ', qw/MODSEQ FLAGS INTERNALDATE BODY.PEEK[]/;
 #############################################################################
 # Synchronize messages
 
-# Update the HIGHESTMODSEQ.
-my $STH_UPDATE_LOCAL_HIGHESTMODSEQ  = $DBH->prepare(q{UPDATE local  SET HIGHESTMODSEQ = ? WHERE idx = ?});
-my $STH_UPDATE_REMOTE_HIGHESTMODSEQ = $DBH->prepare(q{UPDATE remote SET HIGHESTMODSEQ = ? WHERE idx = ?});
-
-# Update the HIGHESTMODSEQ and UIDNEXT.
-my $STH_UPDATE_LOCAL  = $DBH->prepare(q{UPDATE local  SET UIDNEXT = ?, HIGHESTMODSEQ = ? WHERE idx = ?});
-my $STH_UPDATE_REMOTE = $DBH->prepare(q{UPDATE remote SET UIDNEXT = ?, HIGHESTMODSEQ = ? WHERE idx = ?});
-
-# Add a new mailbox.
-my $STH_INSERT_LOCAL  = $DBH->prepare(q{INSERT INTO local  (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)});
-my $STH_INSERT_REMOTE = $DBH->prepare(q{INSERT INTO remote (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)});
-
 # Download some missing UIDs from $source; returns the new allocated UIDs
 sub download_missing($$$@) {
     my $idx = shift;
@@ -1243,10 +1231,30 @@ sub sync_messages($$;$$) {
 
     # don't store the new UIDNEXTs before to avoid downloading these
     # mails again in the event of a crash
-    $STH_UPDATE_LOCAL->execute($lIMAP->get_cache( qw/UIDNEXT HIGHESTMODSEQ/), $idx) or
-        msg('database', "WARNING: Can't update remote UIDNEXT/HIGHESTMODSEQ for $mailbox");
-    $STH_UPDATE_REMOTE->execute($rIMAP->get_cache(qw/UIDNEXT HIGHESTMODSEQ/), $idx) or
-        msg('database', "WARNING: Can't update remote UIDNEXT/HIGHESTMODSEQ for $mailbox");
+
+    state $sth_update_local = $DBH->prepare(q{
+        UPDATE local
+           SET UIDNEXT = ?, HIGHESTMODSEQ = ?
+         WHERE idx = ?
+    });
+    state $sth_update_remote = $DBH->prepare(q{
+        UPDATE remote
+           SET UIDNEXT = ?, HIGHESTMODSEQ = ?
+         WHERE idx = ?
+    });
+
+    my ($lUIDNEXT, $lHIGHESTMODSEQ) = $lIMAP->get_cache(qw/UIDNEXT HIGHESTMODSEQ/);
+    $sth_update_local->bind_param(1, $lUIDNEXT,                        SQL_INTEGER);
+    $sth_update_local->bind_param(2, sprintf("%lld", $lHIGHESTMODSEQ), SQL_BIGINT);
+    $sth_update_local->bind_param(3, $idx,                             SQL_INTEGER);
+    $sth_update_local->execute();
+
+    my ($rUIDNEXT, $rHIGHESTMODSEQ) = $rIMAP->get_cache(qw/UIDNEXT HIGHESTMODSEQ/);
+    $sth_update_remote->bind_param(1, $rUIDNEXT,                        SQL_INTEGER);
+    $sth_update_remote->bind_param(2, sprintf("%lld", $rHIGHESTMODSEQ), SQL_BIGINT);
+    $sth_update_remote->bind_param(3, $idx,                             SQL_INTEGER);
+    $sth_update_remote->execute();
+
     $DBH->commit();
 }
 
@@ -1268,6 +1276,9 @@ sub db_get_cache_by_idx($) {
     $sth->execute();
     my $cache = $sth->fetchrow_hashref();
     die if defined $sth->fetch(); # safety check
+    if (defined $cache) {
+        $cache->{$_} = sprintf("%llu", $cache->{$_}) foreach qw/lHIGHESTMODSEQ rHIGHESTMODSEQ/;
+    }
     return $cache;
 }
 
@@ -1377,12 +1388,12 @@ my %KNOWN_INDEXES;
         $lIMAP->set_cache(mbx_name(local => $row->{mailbox}),
             UIDVALIDITY   => $row->{lUIDVALIDITY},
             UIDNEXT       => $row->{lUIDNEXT},
-            HIGHESTMODSEQ => $row->{lHIGHESTMODSEQ}
+            HIGHESTMODSEQ => sprintf("%llu", $row->{lHIGHESTMODSEQ})
         );
         $rIMAP->set_cache(mbx_name(remote => $row->{mailbox}),
             UIDVALIDITY   => $row->{rUIDVALIDITY},
             UIDNEXT       => $row->{rUIDNEXT},
-            HIGHESTMODSEQ => $row->{rHIGHESTMODSEQ}
+            HIGHESTMODSEQ => sprintf("%llu", $row->{rHIGHESTMODSEQ})
         );
         $KNOWN_INDEXES{$row->{idx}} = 1;
     }
@@ -1416,6 +1427,24 @@ if ($CONFIG{notify}) {
 
 
 sub loop() {
+    state $sth_insert_local = $DBH->prepare(q{
+        INSERT INTO local (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)
+    });
+    state $sth_insert_remote = $DBH->prepare(q{
+        INSERT INTO remote (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)
+    });
+
+    state $sth_update_local_highestmodseq = $DBH->prepare(q{
+        UPDATE local
+           SET HIGHESTMODSEQ = ?
+         WHERE idx = ?
+    });
+    state $sth_update_remote_highestmodseq = $DBH->prepare(q{
+        UPDATE remote
+           SET HIGHESTMODSEQ = ?
+         WHERE idx = ?
+    });
+
     while(@MAILBOXES) {
         if (defined $MAILBOX and ($lIMAP->is_dirty(mbx_name(local => $MAILBOX)) or $rIMAP->is_dirty(mbx_name(remote => $MAILBOX)))) {
             # $MAILBOX is dirty on either the local or remote mailbox
@@ -1430,8 +1459,15 @@ sub loop() {
             select_mbx($IDX, $MAILBOX);
 
             if (!$KNOWN_INDEXES{$IDX}) {
-                $STH_INSERT_LOCAL->execute( $IDX, $lIMAP->uidvalidity($MAILBOX));
-                $STH_INSERT_REMOTE->execute($IDX, $rIMAP->uidvalidity($MAILBOX));
+                my $lUIDVALIDITY = $lIMAP->uidvalidity(mbx_name(local => $MAILBOX));
+                $sth_insert_local->bind_param(1, $IDX,          SQL_INTEGER);
+                $sth_insert_local->bind_param(2, $lUIDVALIDITY, SQL_INTEGER);
+                $sth_insert_local->execute();
+
+                my $rUIDVALIDITY = $rIMAP->uidvalidity(mbx_name(remote => $MAILBOX));
+                $sth_insert_remote->bind_param(1, $IDX,          SQL_INTEGER);
+                $sth_insert_remote->bind_param(2, $rUIDVALIDITY, SQL_INTEGER);
+                $sth_insert_remote->execute();
 
                 # no need to commit before the first mapping (lUID,rUID)
                 $KNOWN_INDEXES{$IDX} = 1;
@@ -1439,10 +1475,15 @@ sub loop() {
             elsif (sync_known_messages($IDX, $MAILBOX)) {
                 # sync updates to known messages before fetching new messages
                 # get_cache is safe after pull_update
-                $STH_UPDATE_LOCAL_HIGHESTMODSEQ->execute( $lIMAP->get_cache('HIGHESTMODSEQ'), $IDX) or
-                    msg('database', "WARNING: Can't update local HIGHESTMODSEQ for $MAILBOX");
-                $STH_UPDATE_REMOTE_HIGHESTMODSEQ->execute($rIMAP->get_cache('HIGHESTMODSEQ'), $IDX) or
-                    msg('database', "WARNING: Can't update remote HIGHESTMODSEQ for $MAILBOX");
+                my $lHIGHESTMODSEQ = sprintf "%lld", $lIMAP->get_cache(qw/HIGHESTMODSEQ/);
+                $sth_update_local_highestmodseq->bind_param(1, $lHIGHESTMODSEQ, SQL_BIGINT);
+                $sth_update_local_highestmodseq->bind_param(2, $IDX,            SQL_INTEGER);
+                $sth_update_local_highestmodseq->execute();
+
+                my $rHIGHESTMODSEQ = sprintf "%lld", $rIMAP->get_cache(qw/HIGHESTMODSEQ/);
+                $sth_update_remote_highestmodseq->bind_param(1, $rHIGHESTMODSEQ, SQL_BIGINT);
+                $sth_update_remote_highestmodseq->bind_param(2, $IDX,            SQL_INTEGER);
+                $sth_update_remote_highestmodseq->execute();
                 $DBH->commit();
             }
             sync_messages($IDX, $MAILBOX);