Twisted deferreds firing in undesired way
Question: I have the following code

    
    
    # logging
    from twisted.python import log
    
    import sys
    
    # MIME Multipart handling
    import email
    import email.mime.application
    import uuid
    
    # IMAP Connection
    from twisted.mail import imap4
    from twisted.internet import protocol
    
    
    #SMTP Sending
    import os.path
    from OpenSSL.SSL import SSLv3_METHOD
    from twisted.internet import ssl
    from twisted.mail.smtp import ESMTPSenderFactory
    from twisted.internet.ssl import ClientContextFactory
    from twisted.internet.defer import Deferred
    from twisted.internet import reactor
    
    #class AccountsManager(object):
    
    
    def connectToIMAPServer(imap_server, username, password):
        factory = IMAP4ClientFactory(username, password, login_insecure = True)
    
        host, port = imap_server.split(":")
    
        # connect to reactor
        if port == '993':
            reactor.connectSSL(host, int(port), factory, ssl.ClientContextFactory())
        else:
            if not port:
                port = 143
            reactor.connectTCP(host, int(port), factory)
    
        d = factory.deferred
        d.addCallback(lambda r: factory.proto)
    
        return d
    
    class IMAP4Client(imap4.IMAP4Client):
        """
            A client with callbacks for greeting messages from an IMAP server.
            """
        greetDeferred = None
    
        def serverGreeting(self, caps):
            self.serverCapabilities = caps
            if self.greetDeferred is not None:
                d, self.greetDeferred = self.greetDeferred, None
                d.callback(self)
    
    class IMAP4ClientFactory(protocol.ClientFactory):
        usedUp = False
    
        protocol = IMAP4Client
    
    
        def __init__(self, username, password, mailbox = "INBOX", login_insecure = False):
            self.ctx = ssl.ClientContextFactory()
    
            self.username = username
            self.password = password
            self.mailbox  = mailbox
            self.login_insecure = login_insecure
    
            self.deferred = Deferred()
    
        def buildProtocol(self, addr):
            """
                Initiate the protocol instance. Since we are building a simple IMAP
                client, we don't bother checking what capabilities the server has. We
                just add all the authenticators twisted.mail has.  Note: Gmail no
                longer uses any of the methods below, it's been using XOAUTH since
                2010.
                """
            assert not self.usedUp
            self.usedUp = True
    
            p = self.protocol(self.ctx)
            p.factory = self
    
            p.greetDeferred = self.deferred
    
            p.registerAuthenticator(imap4.PLAINAuthenticator(self.username))
            p.registerAuthenticator(imap4.LOGINAuthenticator(self.username))
            p.registerAuthenticator(imap4.CramMD5ClientAuthenticator(self.username))
    
            self.deferred.addCallback(self.GreetingCallback)
            self.deferred.addErrback(self.GreetingErrback)
    
            self.proto = p
    
            return p
    
        def GreetingCallback(self, result):
            print "Secure Login"
            auth_d = self.proto.authenticate(self.password)
            auth_d.addCallback(self.AuthenticationCallback)
            auth_d.addErrback(self.AuthenticationErrback)
    
            return auth_d # attach it to the main deferred
    
        def GreetingErrback(self, error):
            log.err(error)
            self.CloseConnection()
            return error
    
        def AuthenticationCallback(self, result):
            print "Selecting Mailbox"
            d = self.proto.examine(self.mailbox)
            return d
    
        def AuthenticationErrback(self, failure):
            if self.login_insecure:
                failure.trap(imap4.NoSupportedAuthentication)
                return self.InsecureLogin()
            else:
                return error
    
        def InsecureLogin(self):
            print "Insecure Login"
            d = self.proto.login(self.username, self.password)
            d.addCallback(self.AuthenticationCallback)
            return d
    
        def CloseConnection(self):
            self.proto.transport.loseConnection()
    
        def clientConnectionFailed(self, connector, reason):
            d, self.deferred = self.deferred, None
            d.errback(reason)
    
    
    class MailServer(object):
        "Manages a server"
    
        size = 0
        used_space = 0
    
    
        def __init__(self, smtp_server, imap_server, username, password):
            self.smtp_server, self.smtp_port = smtp_server.split(":")
            self.imap_server, self.imap_port = imap_server.split(":")
            self.username = username
            self.password = password
    
            self.imap_connection = IMAP4ClientFactory(username, password)
    
    
        def upload_data(self, data):
            """
            Uploads data to email server returns deferred that will return with the imap uid
            """
    
            # Create a text/plain message
            id = str(uuid.uuid4()).upper()
    
            msg = email.mime.Multipart.MIMEMultipart()
            msg['Subject'] = 'GMA ID: %s' % id
            msg['From'] = self.email_address
            msg['To'] = self.email_address
    
            # The main body is just another attachment
            body = email.mime.Text.MIMEText("GMA ID: %s" % (self.uuid_id))
            msg.attach(body)
            att = email.mime.application.MIMEApplication(data,_subtype="raw")
            att.add_header('Content-Disposition','attachment',filename = os.path.basename(self.filename))
            msg.attach(att)
    
            # Create a context factory which only allows SSLv3 and does not verify
            # the peer's certificate.
            contextFactory = ClientContextFactory()
            contextFactory.method = SSLv3_METHOD
    
            d = Deferred()
    
            mime_obj = StringIO(str(msg))
    
            senderFactory = ESMTPSenderFactory(
                                               self.username,
                                               self.password,
                                               self.email_address,
                                               self.email_address,
                                               mime_obj,
                                               d,
                                               contextFactory=contextFactory)
    
            d.addCallback(lambda r: self.email_sent(id, int(self.parts)) )
            d.addErrback(self.email_error)
    
            reactor.connectTCP(self.smtp_server, self.smtp_port, senderFactory)
    
            d.addCallback(self.upload_success, *args, **kw)
            d.addErrback(self.upload_error, 1)
    
            return d
    
    
        def upload_success(self, result):
            print "upload was succesful!"
    
        def upload_error(self, result):
            print "upload error"
    
        def download_data(self, uid):
            """
            Downloads data from the email server returns a deferred that will return with the data
            """
            print "uid"
    
    
    if __name__ == "__main__":
        log.startLogging(sys.stdout)
        d = connectToIMAPServer("imap.gmail.com:993", "username", "password")
        def f(s):
            print s
    
    
        d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
        d.addCallback(lambda r: f("These are fired before the auth and examine callbacks, why?"))
        reactor.run()
    

The class is suppose to handle logging in and selecting a mailbox and nicely
return a IMAP proto ready to use however the two callbacks at the bottom are
fired before the other ones, I get why, the callbacks are added before the
other ones because buildProtocol hasn't been called yet so what is the best
way to handle this, just have a dummy callback added in **init** that "holds"
the first spot?


Answer: 
    from twisted.internet.endpoints import TCP4ClientEndpoint
    d = TCP4ClientEndpoint(reactor, host, int(port)).connect(factory)
    

and

    
    
    d.addCallback(lambda r: factory.deferred)    
    

instead of

    
    
    d = factory.deferred
    

in `connectToIMAPServer` should do it - your `factory.deferred` will be
returned only after protocol is ready. ([Twisted Documentation on writing
clients](http://twistedmatrix.com/documents/11.1.0/core/howto/clients.html))

