On Thu, Feb 02, 2012 at 01:11:24PM +0100, Manel Gimeno Zaragozá wrote:
What version?
> Everything is working fine between Master & Replica until Replica goes down
> (maintenance or whatever). Once is down, in master there are still
> connections and modification. If I bring up Replica the changes made during
> the "blackout" are no fully transmitted to it. Some of the are transmitted,
> but not all.
>
> Which is the best way to resincronize the missing parts to Replica?
sync_client -r -f $file for each file in the conf/sync/ directory.
> I was wordering to rsync imap folders (/var/spool/imap & /var/lib/imap), and
> It works fine in my test environment, but in produccion it could be GB to
> sincronize and I think this is not optimum.
No, that's awful. Don't do that.
> Other option was "sync_client -l -u <all_user>" but I'm guessing that in
> production I will have the same problem due to the amount of data to
> sincronize.
It's not too bad. We run it occasionally if we suspect things have
got confused for other reasons.
> On the other hand, I have also doubts about how "sync_client -r" and
> "sync_server" are initiated. In my test machines I should run manually them
> each time I start cyrus-imapd, I though that it could be automatic one I
> start the service cyrus-imapd, but it is not.
I have attached the script that we run. It has lots of hooks into
our own systems of course.
I would LIKE to make something more generic that's part of Cyrus
itself, rather than having external tooling. Of course, our external
tooling is a bit special-cased as well. But at least having a basic
"keep trying to replicate all the records" would be good.
Bron.
#!/usr/bin/perl -w
BEGIN { do "/home/mod_perl/hm/ME/FindLibs.pm"; }
use strict;
use warnings;
use ME::FMVars;
use ME::ImapStore;
use ME::ImapSlot;
use ME::Machine;
use ME::Notify qw(NotifyAdmins);
use IO::LockedFile;
use IO::File;
use Getopt::Std;
use Data::Dumper;
my %Opts;
getopts('Cvnl:r:fa', \%Opts);
my $MachineName = ME::Machine->Name();
my $RateLimit = 5000;
my $OneoffLimit = $Opts{l} || $RateLimit;
# easy way to suppress cron runs
unless ($Opts{f}) {
exit 0 if -e "/etc/nomonitorsync";
}
foreach my $Slot (ME::Machine->ImapSlots()) {
my $SlotName = $Slot->Name();
unless ($Opts{a} or $Slot->IsMaster()) {
print "$SlotName: not master\n" if $Opts{v};
next;
}
my @OtherSlots = $Slot->OtherSlots();
unless (@OtherSlots) {
print "$SlotName: not replicated\n" if $Opts{v};
next;
}
my $ConfDir = $Slot->CyrusConfigPath();
my %dirs;
my $ImapdConf = $Slot->ImapdConf();
if ($ImapdConf->{sync_log_channels}) {
foreach my $channel (split / /, $ImapdConf->{sync_log_channels}) {
$dirs{$channel} = "$ConfDir/sync/$channel";
}
my %channels = map { $_ => 1 } split / /, $ImapdConf->{sync_log_channels};
foreach my $osl (map { $_->Name() } @OtherSlots) {
delete $channels{$osl};
}
delete $channels{oslotest};
if (keys %channels) {
warn "$SlotName: extra channels in imapd configuration file - "
.join(" ", keys %channels) . "\n";
}
} elsif ($ImapdConf->{sync_host}) {
$dirs{''} = "$ConfDir/sync";
}
opendir(DH, "$ConfDir/sync/");
while(my $item = readdir(DH)) {
next if $item =~ m/^\./;
next unless -d "$ConfDir/sync/$item";
next if $dirs{$item}; # channel exists
warn "$SlotName: extra channel $item\n";
next unless $Opts{C};
warn "rm -rf $ConfDir/sync/$item\n";
system("rm -rf $ConfDir/sync/$item\n");
}
my $DH;
foreach my $channel (sort keys %dirs) {
print "Doing slot $SlotName/$channel\n" if $Opts{v};
my $dir = $dirs{$channel};
next unless -d $dir; # no logs!
if (-f "$dir/shutdown") {
print "$SlotName/$channel: shutdown file exists, skipping\n";
next;
}
my $OtherSlot;
if ($channel) {
$OtherSlot = eval { ME::ImapSlot->new($channel) };
} else {
# There is only one replica in this case
$OtherSlot = $OtherSlots[0];
}
next unless $OtherSlot;
unless ($OtherSlot->IsRunning()) {
print "$SlotName/$channel: ignoring, replica is down\n" if $Opts{v};
next;
}
if ($Opts{r} and not $OtherSlot->Machine->HasRole($Opts{r})) {
print "$SlotName/$channel: not syncing, doesn't have role $Opts{r}\n";
next;
}
my @rest;
@rest = (-n => $channel) if $channel;
my $Lock = IO::LockedFile->new({block => 0}, ">$dir/monitorsync.lock");
unless ($Lock) {
print "$SlotName/$channel: already locked, skipping\n" if $Opts{v};
next;
}
my @pids = get_pids($SlotName, $channel);
my @ran;
next unless opendir($DH, $dir);
while (my $item = readdir($DH)) {
if ($item eq 'log' and not @pids) {
$item = "slog-$$";
print ("renaming log to $item\n");
rename("$dir/log", "$dir/$item");
}
next unless $item =~ m/^(?:s\d*)?log-(\d+)$/;
my $pid = $1 || '';
# check if pid exists
if ($pid and my $fh = IO::File->new("</proc/$pid/cmdline")) {
local $/;
my $cmdline = <$fh>;
$fh->close();
if ($cmdline =~ m/sync_client.*$SlotName/s) {
print "$SlotName/$channel: Skipping log file log-$pid, process is running\n" if $Opts{v};
next;
}
}
print "$SlotName/$channel: Syncing file $item\n" if $Opts{v};
# returns an empty string on success
my $res = $Slot->RunCommand({RateLimit => $RateLimit},
'sync_client', '-o', '-r',
'-f' => "$dir/$item", @rest);
# failure
if ($? or $res =~ m/\S/) {
print "$SlotName: Failed $channel $item, notifying ($res, $?)\n" if $Opts{v};
NotifyAdmins('email', "$MachineName/$SlotName/$channel sync_client failed on $item", "$res (error: $?)");
}
# success :)
else {
print "$SlotName/$channel: Done $item, deleting\n" if $Opts{v};
push @ran, $item;
unlink("$dir/$item");
}
}
closedir($DH);
if (@ran) {
my $num = @ran;
NotifyAdmins('email', "$MachineName/$SlotName/$channel sync_client ran $num leftover logs", join("\n", @ran)) if $Opts{n};
}
unless (@pids) {
# start a new one
print "$SlotName: Starting a new sync_client, old one gone away\n" if $Opts{v};
NotifyAdmins('email', "$MachineName/$SlotName/$channel sync_client missing, starting a new one") if $Opts{n};
$Slot->RunCommand({Daemon => 1, RateLimit => $RateLimit},
'sync_client', '-r', '-o', '-v', @rest);
}
}
}
print "Finished\n" if $Opts{v};
sub get_pids {
my $SlotName = shift;
my $Channel = shift;
my @res;
if (open(my $FH, "ps axww |")) {
while (<$FH>) {
next unless m/sync_client/;
next if m/ -f/; # specific file, we don't want that
next unless m/ -r/; # needs to be rolling
next unless m/imapd-$SlotName/;
next unless (not $Channel or m/-n $Channel/);
next unless m/^\s*(\d+)/;
my $Pid = $1;
push @res, $Pid;
}
}
return @res;
}
----
Cyrus Home Page: http://www.cyrusimap.org/
List Archives/Info: http://lists.andrew.cmu.edu/pipermail/info-cyrus/
