Author: fhanik Date: Mon May 22 20:41:04 2006 New Revision: 408823 URL: http://svn.apache.org/viewvc?rev=408823&view=rev Log: Defined the algorithm for leadership election, need to create a state diagram so that the implementation doesn't become a clutter
Modified: tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/AbsoluteOrder.java tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/membership/Membership.java tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/util/Arrays.java Modified: tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/AbsoluteOrder.java URL: http://svn.apache.org/viewvc/tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/AbsoluteOrder.java?rev=408823&r1=408822&r2=408823&view=diff ============================================================================== --- tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/AbsoluteOrder.java (original) +++ tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/AbsoluteOrder.java Mon May 22 20:41:04 2006 @@ -27,7 +27,7 @@ * @see org.apache.catalina.tribes.Member */ public class AbsoluteOrder { - protected static AbsoluteComparator comp = new AbsoluteComparator(); + public static final AbsoluteComparator comp = new AbsoluteComparator(); protected AbsoluteOrder() { super(); Modified: tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java URL: http://svn.apache.org/viewvc/tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java?rev=408823&r1=408822&r2=408823&view=diff ============================================================================== --- tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java (original) +++ tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/group/interceptors/NonBlockingCoordinator.java Mon May 22 20:41:04 2006 @@ -26,6 +26,8 @@ import org.apache.catalina.tribes.util.UUIDGenerator; import org.apache.catalina.tribes.group.AbsoluteOrder; import org.apache.catalina.tribes.util.Arrays; +import org.apache.catalina.tribes.io.ChannelData; +import org.apache.catalina.tribes.Channel; /** * <p>Title: NonBlockingCoordinator</p> @@ -33,39 +35,215 @@ * <p>Description: Implementation of a simple coordinator algorithm.</p> * <p>This algorithm is non blocking meaning it allows for transactions while the coordination phase is going on * </p> - * <p>Implementation based on ideas fetched from <a href="http://www.cs.chalmers.se/~hanssv/">Hans Svensson</a></p> + * <p>This implementation is based on a home brewed algorithm that uses the AbsoluteOrder of a membership + * to pass a token ring of the current membership.<br> + * This is not the same as just using AbsoluteOrder! Consider the following scenario:<br> + * Nodes, A,B,C,D,E on a network, in that priority. AbsoluteOrder will only work if all + * nodes are receiving pings from all the other nodes. + * meaning, that node{i} receives pings from node{all}-node{i}<br> + * but the following could happen if a multicast problem occurs. + * A has members {B,C,D}<br> + * B has members {A,C}<br> + * C has members {D,E}<br> + * D has members {A,B,C,E}<br> + * E has members {A,C,D}<br> + * Because the default Tribes membership implementation, relies on the multicast packets to + * arrive at all nodes correctly, there is nothing guaranteeing that it will.<br> + * <br> + * To best explain how this algorithm works, lets take the above example: + * For simplicity we assume that a send operation is O(1) for all nodes, although this algorithm will work + * where messages overlap, as they all depend on absolute order<br> + * Scenario 1: A,B,C,D,E all come online at the same time + * Eval phase, A thinks of itself as leader, B thinks of A as leader, + * C thinks of itself as leader, D,E think of A as leader<br> + * Token phase:<br> + * (1) A sends out a message X{A-ldr, A-src, mbrs-A,B,C,D} to B where X is the id for the message(and the view)<br> + * (1) C sends out a message Y{C-ldr, C-src, mbrs-C,D,E} to D where Y is the id for the message(and the view)<br> + * (2) B receives X{A-ldr, A-src, mbrs-A,B,C,D}, sends X{A-ldr, A-src, mbrs-A,B,C,D} to C <br> + * (2) D receives Y{C-ldr, C-src, mbrs-C,D,E} D is aware of A,B, sends Y{A-ldr, C-src, mbrs-A,B,C,D,E} to E<br> + * (3) C receives X{A-ldr, A-src, mbrs-A,B,C,D}, sends X{A-ldr, A-src, mbrs-A,B,C,D,E} to D<br> + * (3) E receives Y{A-ldr, C-src, mbrs-A,B,C,D,E} sends Y{A-ldr, C-src, mbrs-A,B,C,D,E} to A<br> + * (4) D receives X{A-ldr, A-src, mbrs-A,B,C,D,E} sends sends X{A-ldr, A-src, mbrs-A,B,C,D,E} to A<br> + * (4) A receives Y{A-ldr, C-src, mbrs-A,B,C,D,E}, holds the message, add E to its list of members<br> + * (5) A receives X{A-ldr, A-src, mbrs-A,B,C,D,E} <br> + * At this point, the state looks like<br> + * A - {A-ldr, mbrs-A,B,C,D,E, id=X}<br> + * B - {A-ldr, mbrs-A,B,C,D, id=X}<br> + * C - {A-ldr, mbrs-A,B,C,D,E, id=X}<br> + * D - {A-ldr, mbrs-A,B,C,D,E, id=X}<br> + * E - {A-ldr, mbrs-A,B,C,D,E, id=Y}<br> + * <br> + * A message doesn't stop until it reaches its original sender, unless its dropped by a higher leader. + * As you can see, E still thinks the viewId=Y, which is not correct. But at this point we have + * arrived at the same membership and all nodes are informed of each other.<br> + * To synchronize the rest we simply perform the following check at A when A receives X:<br> + * Original X{A-ldr, A-src, mbrs-A,B,C,D} == Arrived X{A-ldr, A-src, mbrs-A,B,C,D,E}<br> + * Since the condition is false, A, will resend the token, and A sends X{A-ldr, A-src, mbrs-A,B,C,D,E} to B + * When A receives X again, the token is complete. + * </p> + * <p> + * Lets assume that C1 arrives, C1 has lower priority than C, but higher priority than D.<br> + * Lets also assume that C1 sees the following view {B,D,E}<br> + * C1 sends Z{B-ldr, C-src, mbrs-B,C1,D,E} to D<br> + * D receives Z{B-ldr, C-src, mbrs-B,C1,D,E} sends Z{A-ldr, D-src, mbrs-A,B,C,C1,D,E} to E<br> + * Once the message reaches A, A will issue a new view and send a new message<br> + * A view is not accepted by a member unless ldr==src in the token.<br> + * </p> + * <p> + * Lets assume that A0 arrives A0 being higher than A.<br> + * Lets also assume that A0 sees view {B,D,E}<br> + * A0 will issue a similar view statement and the same scenario as above will happen.<br> + * If A0 sees {A,B,C,D} it simply sends the message to A rather than B. + * </p> + * <p>If we wanted to ensure that the view gets implemented at all nodes at the same time, + * ie, implementing a blocking coordinator, we would simply require that each view, before it gets installed + * has to receive a VIEW_CONF message. + * * <p>Ideally, the interceptor below this one would be the TcpFailureDetector to ensure correct memberships</p> * + * <p>The example above, of course can be simplified with a finite statemachine:<br> + * But I suck at writing state machines, my head gets all confused. One day I will document this algorithm though.<br> + * Maybe I'll do a state diagram :) + * </p> + * * @author Filip Hanik * @version 1.0 - * @todo - * when sending a HALT message, btw, only the highest in the membership group will do that - * allow for some time to pass, incase there is a higher member around - * preferrably, place a mcast interceptor below, so that we can mcast this sucker + * + * + * */ public class NonBlockingCoordinator extends ChannelInterceptorBase { - protected Membership membership = null; - - protected static byte[] NBC_HEADER = new byte[] {-86, 38, -34, -29, -98, 90, 65, 63, -81, -122, -6, -110, 99, -54, 13, 63}; - protected static byte[] NBC_REQUEST = new byte[] {-55, -37, 18, -52, -105, 107, 72, 40, -122, 29, 70, -19, -74, 123, 61, 110}; - protected static byte[] NBC_REPLY = new byte[] {6, -15, 14, 23, -96, 106, 78, 124, -94, -122, -85, 31, 88, 21, 126, 20}; - - protected static byte[] NBC_HALT = new byte[] {12, -28, 85, -97, -102, -35, 74, 9, -65, -78, -83, -84, -29, -70, -23, -15}; - protected static byte[] NBC_ACK = new byte[] {12, -49, 117, -70, 77, 52, 65, -91, -93, -110, 37, 34, -28, -127, 26, 18}; - protected static byte[] NBC_NORM = new byte[] {34, -110, 83, 118, -109, -55, 67, -27, -97, -94, -84, -72, -82, -114, 65, 81}; - protected static byte[] NBC_NOTNORM = new byte[] {125, -70, -102, -125, -78, -39, 73, -80, -89, 84, 120, 83, 25, 42, 88, -76}; - protected static byte[] NBC_LDR = new byte[] {97, 31, -23, 30, -42, -72, 72, 116, -97, 7, 112, 25, 82, -96, -87, -48}; - protected static byte[] NBC_HASLDR = new byte[] {93, -80, -88, -58, -127, 21, 76, -90, -89, 77, 58, 25, -55, 65, -1, -83}; - protected static byte[] NBC_ISLDR = new byte[] {104, -95, -92, -42, 114, -36, 71, -19, -79, 20, 122, 101, -1, -48, -49, 30}; + /** + * header for a coordination message + */ + protected static final byte[] COORD_HEADER = new byte[] {-86, 38, -34, -29, -98, 90, 65, 63, -81, -122, -6, -110, 99, -54, 13, 63}; + /** + * Coordination request + */ + protected static final byte[] COORD_REQUEST = new byte[] {104, -95, -92, -42, 114, -36, 71, -19, -79, 20, 122, 101, -1, -48, -49, 30}; + /** + * Coordination confirmation, for blocking installations + */ + protected static final byte[] COORD_CONF = new byte[] {67, 88, 107, -86, 69, 23, 76, -70, -91, -23, -87, -25, -125, 86, 75, 20}; protected Member coordinator = null; + + protected Membership view = null; + protected Membership suggestedview = null; + protected UniqueId viewId; + protected UniqueId suggestedviewId; + + protected boolean started = false; + protected final int startsvc = 0xFFFF; + + protected Object electionMutex = new Object(); + protected boolean runningElection = false; public NonBlockingCoordinator() { super(); } + public void start(int svc) throws ChannelException { + try { + halt(); + if ( started ) return; + super.start(startsvc); + started = true; + + }finally { + release(); + } + //coordination can happen before this line of code executes + Member local = getLocalMember(false); + if (local != null && coordinator == null) coordinator = local; + } + + public void stop(int svc) throws ChannelException { + try { + halt(); + if ( !started ) return; + super.stop(startsvc); + started = false; + }finally { + release(); + } + this.coordinator = null; + } + + public void elect() { + synchronized (electionMutex) { + try { + Member[] mbrs = super.getMembers(); + //no members, exit + if ( mbrs.length == 0 ) return; + AbsoluteOrder.absoluteOrder(mbrs); + MemberImpl local = (MemberImpl)getLocalMember(false); + //I'm not the higest, exit + if ( !local.equals(mbrs[0]) ) return; + //I'm already running an election + if ( suggestedview.hasMembers() ) return; + //create a suggestedview + suggestedview.addMember((MemberImpl)local); + Arrays.fill(suggestedview,mbrs); + suggestedviewId = new UniqueId(UUIDGenerator.randomUUID(true)); + CoordinationMessage msg = new CoordinationMessage(local,local,suggestedview.getMembers(),suggestedviewId); + for (int i=0; i<mbrs.length; i++ ) { + try { + sendMessage(msg,mbrs[i]); + break; + } catch ( ChannelException x ) { + log.error("Unable to send election message, trying next node.",x); + } + } + halt(); + } finally { + //dont release, election running + //release will happen on processCoordMessage + } + } + } + + protected void viewChange(UniqueId viewId, Member[] view) { + //invoke any listeners + } + + protected void processCoordMessage(CoordinationMessage msg, Member sender) { + synchronized (electionMutex) { + MemberImpl local = (MemberImpl) getLocalMember(false); + if (suggestedviewId != null) { + //we are running our own election + if (suggestedviewId.equals(msg.getId())) { + //we received our own token + Member[] suggested = suggestedview.getMembers(); + Member[] received = msg.getMembers(); + if (Arrays.sameMembers(suggested,received) ) { + //did the view change + suggestedviewId = null; + suggestedview.reset(); + viewChange(msg.getId(),received); + } else { + //view or leadership changed + + } + } + } else { + + } + } + } + + + protected void sendMessage(CoordinationMessage msg, Member dest) throws ChannelException { + ChannelData data = new ChannelData(UUIDGenerator.randomUUID(false),msg.getBuffer(),System.currentTimeMillis()); + data.setOptions(Channel.SEND_OPTIONS_USE_ACK); + Member[] destination = new Member[] {dest}; + sendMessage(destination,data,null); + } + + + public Member getCoordinator() { return coordinator; } @@ -76,8 +254,8 @@ } public void messageReceived(ChannelMessage msg) { - if ( Arrays.contains(msg.getMessage().getBytesDirect(),0,NBC_HEADER,0,NBC_HEADER.length) ) { - receiveNBC(msg.getMessage()); + if ( Arrays.contains(msg.getMessage().getBytesDirect(),0,COORD_HEADER,0,COORD_HEADER.length) ) { + processCoordMessage(new CoordinationMessage(msg.getMessage()),msg.getAddress()); } else { super.messageReceived(msg); } @@ -90,8 +268,6 @@ public void memberAdded(Member member) { try { halt(); - if (membership == null) setupMembership(); - membership.addMember( (MemberImpl) member); }finally { release(); } @@ -101,8 +277,7 @@ public void memberDisappeared(Member member) { try { halt(); - if (membership == null) setupMembership(); - membership.removeMember( (MemberImpl) member); + if ( started ) elect(); }finally { release(); } @@ -117,8 +292,8 @@ * has members */ public boolean hasMembers() { - if ( membership == null ) setupMembership(); - return membership.hasMembers(); + if ( view == null ) setupMembership(); + return view.hasMembers(); } /** @@ -126,9 +301,8 @@ * @return all members or empty array */ public Member[] getMembers() { - if ( membership == null ) setupMembership(); - Member[] members = membership.getMembers(); - AbsoluteOrder.absoluteOrder(members); + if ( view == null ) setupMembership(); + Member[] members = view.getMembers(); return members; } @@ -138,8 +312,8 @@ * @return Member */ public Member getMember(Member mbr) { - if ( membership == null ) setupMembership(); - return membership.getMember(mbr); + if ( view == null ) setupMembership(); + return view.getMember(mbr); } /** @@ -149,13 +323,14 @@ */ public Member getLocalMember(boolean incAlive) { Member local = super.getLocalMember(incAlive); - if ( membership == null && (local != null)) setupMembership(); + if ( view == null && (local != null)) setupMembership(); return local; } protected synchronized void setupMembership() { - if ( membership == null ) { - membership = new Membership((MemberImpl)super.getLocalMember(true)); + if ( view == null || suggestedview == null ) { + view = new Membership((MemberImpl)super.getLocalMember(true)); + suggestedview = new Membership((MemberImpl)super.getLocalMember(true)); } } @@ -180,38 +355,123 @@ } - /** - * A message is:<br> - * HEADER, REQUEST|REPLY, ID, MSG, SOURCE_LEN, SOURCE, PAYLOAD_LEN, PAYLOAD - * @param type byte[] - either NBC_REQUEST or NBC_REPLY - * @param msg byte[] - NBC_HALT, NBC_ACK, NBC_NORM, NBC_NOTNORM, NBC_LDR - */ - protected UniqueId createNBCMessage(XByteBuffer buf, byte[] type, byte[] msg, byte[] payload) { - UniqueId id = new UniqueId(UUIDGenerator.randomUUID(true)); - Member local = getLocalMember(false); - byte[] ldata = ((MemberImpl)local).getData(false,false); - buf.reset(); - buf.append(NBC_HEADER,0,NBC_HEADER.length); - buf.append(type,0,type.length); - buf.append(id.getBytes(),0,id.getBytes().length); - buf.append(msg,0,msg.length); - buf.append(ldata.length); - buf.append(ldata,0,ldata.length); - buf.append(payload.length); - buf.append(payload,0,payload.length); - return id; - } - - protected void receiveNBC(XByteBuffer buf) { + public static class CoordinationMessage { + //X{A-ldr, A-src, mbrs-A,B,C,D} + protected XByteBuffer buf; + protected MemberImpl leader; + protected MemberImpl source; + protected MemberImpl[] view; + protected UniqueId id; + + public CoordinationMessage(XByteBuffer buf) { + this.buf = buf; + } + + public CoordinationMessage(MemberImpl leader, + MemberImpl source, + MemberImpl[] view, + UniqueId id) { + this.buf = new XByteBuffer(4096,false); + this.leader = leader; + this.source = source; + this.view = view; + this.id = id; + this.write(); + } + + + public byte[] getHeader() { + return NonBlockingCoordinator.COORD_HEADER; + } + + public MemberImpl getLeader() { + if ( leader == null ) parse(); + return leader; + } + + public MemberImpl getSource() { + if ( source == null ) parse(); + return source; + } + + public UniqueId getId() { + if ( id == null ) parse(); + return id; + } + + public MemberImpl[] getMembers() { + if ( view == null ) parse(); + return view; + } + + public XByteBuffer getBuffer() { + return this.buf; + } + + public void parse() { + //header + int offset = 16; + //leader + int ldrLen = buf.toInt(buf.getBytesDirect(),offset); + offset += 4; + byte[] ldr = new byte[ldrLen]; + System.arraycopy(buf.getBytesDirect(),offset,ldr,0,ldrLen); + leader = MemberImpl.getMember(ldr); + offset += ldrLen; + //source + int srcLen = buf.toInt(buf.getBytesDirect(),offset); + offset += 4; + byte[] src = new byte[srcLen]; + System.arraycopy(buf.getBytesDirect(),offset,src,0,srcLen); + source = MemberImpl.getMember(src); + offset += srcLen; + //view + int mbrCount = buf.toInt(buf.getBytesDirect(),offset); + offset += 4; + view = new MemberImpl[mbrCount]; + for (int i=0; i<view.length; i++ ) { + int mbrLen = buf.toInt(buf.getBytesDirect(),offset); + offset += 4; + byte[] mbr = new byte[mbrLen]; + System.arraycopy(buf.getBytesDirect(), offset, mbr, 0, mbrLen); + view[i] = MemberImpl.getMember(mbr); + offset += mbrLen; + } + //id + this.id = new UniqueId(buf.getBytesDirect(),offset,16); + offset += 16; + } + + public void write() { + buf.reset(); + //header + buf.append(COORD_HEADER,0,COORD_HEADER.length); + //leader + byte[] ldr = leader.getData(false,false); + buf.append(ldr.length); + buf.append(ldr,0,ldr.length); + ldr = null; + //source + byte[] src = source.getData(false,false); + buf.append(src.length); + buf.append(src,0,src.length); + src = null; + //view + buf.append(view.length); + for (int i=0; i<view.length; i++ ) { + byte[] mbr = view[i].getData(false,false); + buf.append(mbr.length); + buf.append(mbr,0,mbr.length); + } + //id + buf.append(id.getBytes(),0,id.getBytes().length); + } + + + } - public void start(int svc) throws ChannelException { - super.start(svc); - //coordination can happen before this line of code executes - Member local = getLocalMember(false); - if (local != null && coordinator == null) coordinator = local; - } Modified: tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/membership/Membership.java URL: http://svn.apache.org/viewvc/tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/membership/Membership.java?rev=408823&r1=408822&r2=408823&view=diff ============================================================================== --- tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/membership/Membership.java (original) +++ tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/membership/Membership.java Mon May 22 20:41:04 2006 @@ -114,12 +114,14 @@ public synchronized MbrEntry addMember(MemberImpl member) { synchronized (members) { MbrEntry entry = new MbrEntry(member); - map.put(member,entry); - MemberImpl results[] = new MemberImpl[members.length + 1]; - for (int i = 0; i < members.length; i++) results[i] = members[i]; - results[members.length] = member; - members = results; - Arrays.sort(members, memberComparator); + if (!map.containsKey(member) ) { + map.put(member, entry); + MemberImpl results[] = new MemberImpl[members.length + 1]; + for (int i = 0; i < members.length; i++) results[i] = members[i]; + results[members.length] = member; + members = results; + Arrays.sort(members, memberComparator); + } return entry; } } @@ -139,10 +141,8 @@ break; } } - if (n < 0) - return; - MemberImpl results[] = - new MemberImpl[members.length - 1]; + if (n < 0) return; + MemberImpl results[] = new MemberImpl[members.length - 1]; int j = 0; for (int i = 0; i < members.length; i++) { if (i != n) @@ -151,7 +151,7 @@ members = results; } } - + /** * Runs a refresh cycle and returns a list of members that has expired. * This also removes the members from the membership, in such a way that Modified: tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/util/Arrays.java URL: http://svn.apache.org/viewvc/tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/util/Arrays.java?rev=408823&r1=408822&r2=408823&view=diff ============================================================================== --- tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/util/Arrays.java (original) +++ tomcat/container/tc5.5.x/modules/groupcom/src/share/org/apache/catalina/tribes/util/Arrays.java Mon May 22 20:41:04 2006 @@ -17,6 +17,13 @@ import org.apache.catalina.tribes.UniqueId; import org.apache.catalina.tribes.ChannelMessage; +import org.apache.catalina.tribes.Member; +import org.apache.catalina.tribes.group.AbsoluteOrder; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Comparator; +import org.apache.catalina.tribes.membership.Membership; +import org.apache.catalina.tribes.membership.MemberImpl; /** * @author Filip Hanik @@ -53,12 +60,45 @@ return buf.toString(); } + public static int add(int[] data) { + int result = 0; + for (int i=0;i<data.length; i++ ) result += data[i]; + return result; + } + public static UniqueId getUniqudId(ChannelMessage msg) { return new UniqueId(msg.getUniqueId()); } public static UniqueId getUniqudId(byte[] data) { return new UniqueId(data); + } + + public static boolean equals(Object[] o1, Object[] o2) { + boolean result = o1.length == o2.length; + if ( result ) for (int i=0; i<o1.length && result; i++ ) result = o1[i].equals(o2[i]); + return result; + } + + public static boolean sameMembers(Member[] m1, Member[] m2) { + AbsoluteOrder.absoluteOrder(m1); + AbsoluteOrder.absoluteOrder(m2); + return equals(m1,m2); + } + + public static Member[] merge(Member[] m1, Member[] m2) { + AbsoluteOrder.absoluteOrder(m1); + AbsoluteOrder.absoluteOrder(m2); + ArrayList list = new ArrayList(java.util.Arrays.asList(m1)); + for (int i=0; i<m2.length; i++) if ( !list.contains(m2[i]) ) list.add(m2[i]); + Member[] result = new Member[list.size()]; + list.toArray(result); + AbsoluteOrder.absoluteOrder(result); + return result; + } + + public static void fill(Membership mbrship, Member[] m) { + for (int i=0; i<m.length; i++ ) mbrship.addMember((MemberImpl)m[i]); } --------------------------------------------------------------------- To unsubscribe, e-mail: [EMAIL PROTECTED] For additional commands, e-mail: [EMAIL PROTECTED]