Give sudo access to celery workers
Question: I have a python program that needs to executed like this;

    
    
    $ sudo python my_awesome_program.py
    

Now I want to run thousands of instances of this program using celery, of
course with different parameters. The Problem is that while celery tries to
execute my program it fails and the reason is that it doesn't have `sudo`
access.

How can I give my celery workers the power of `sudo` to run this program ?  
I already tried to give sudo access to my user. Changed celery service owner
etc.

May be a stupid question, but I am lost.  
  
P.S  
  
**task.py**

    
    
    from celery import Celery
    import os
    import socket
    import struct
    import select
    import time
    import logging
    # Global variables
    broker = "redis://%s:%s" % ("127.0.0.1", '6379')
    app = Celery('process_ips', broker=broker)
    logging.basicConfig(filename="/var/log/celery_ping.log", level=logging.INFO)
    # From /usr/include/linux/icmp.h; your milage may vary.
    ICMP_ECHO_REQUEST = 8  # Seems to be the same on Solaris.
    
    
    def checksum(source_string):
        """
        I'm not too confident that this is right but testing seems
        to suggest that it gives the same answers as in_cksum in ping.c
        """
        sum = 0
        countTo = (len(source_string) / 2) * 2
        count = 0
        while count < countTo:
            thisVal = ord(source_string[count + 1]) * \
                256 + ord(source_string[count])
            sum = sum + thisVal
            sum = sum & 0xffffffff  # Necessary?
            count = count + 2
    
        if countTo < len(source_string):
            sum = sum + ord(source_string[len(source_string) - 1])
            sum = sum & 0xffffffff  # Necessary?
    
        sum = (sum >> 16) + (sum & 0xffff)
        sum = sum + (sum >> 16)
        answer = ~sum
        answer = answer & 0xffff
    
        # Swap bytes. Bugger me if I know why.
        answer = answer >> 8 | (answer << 8 & 0xff00)
    
        return answer
    
    
    def receive_one_ping(my_socket, ID, timeout):
        """
        receive the ping from the socket.
        """
        timeLeft = timeout
        while True:
            startedSelect = time.time()
            whatReady = select.select([my_socket], [], [], timeLeft)
            howLongInSelect = (time.time() - startedSelect)
            if whatReady[0] == []:  # Timeout
                return
    
            timeReceived = time.time()
            recPacket, addr = my_socket.recvfrom(1024)
            icmpHeader = recPacket[20:28]
            type, code, checksum, packetID, sequence = struct.unpack(
                "bbHHh", icmpHeader
            )
            if packetID == ID:
                bytesInDouble = struct.calcsize("d")
                timeSent = struct.unpack("d", recPacket[28:28 + bytesInDouble])[0]
                return timeReceived - timeSent
    
            timeLeft = timeLeft - howLongInSelect
            if timeLeft <= 0:
                return
    
    
    def send_one_ping(my_socket, dest_addr, ID):
        """
        Send one ping to the given >dest_addr<.
        """
        dest_addr = socket.gethostbyname(dest_addr)
    
        # Header is type (8), code (8), checksum (16), id (16), sequence (16)
        my_checksum = 0
    
        # Make a dummy heder with a 0 checksum.
        header = struct.pack("bbHHh", ICMP_ECHO_REQUEST, 0, my_checksum, ID, 1)
        bytesInDouble = struct.calcsize("d")
        data = (192 - bytesInDouble) * "Q"
        data = struct.pack("d", time.time()) + data
    
        # Calculate the checksum on the data and the dummy header.
        my_checksum = checksum(header + data)
    
        # Now that we have the right checksum, we put that in. It's just easier
        # to make up a new header than to stuff it into the dummy.
        header = struct.pack(
            "bbHHh", ICMP_ECHO_REQUEST, 0, socket.htons(my_checksum), ID, 1
        )
        packet = header + data
        my_socket.sendto(packet, (dest_addr, 1))  # Don't know about the 1
    
    
    def do_one(dest_addr, timeout):
        """
        Returns either the delay (in seconds) or none on timeout.
        """
        logging.info('Called do_one Line 105')
        icmp = socket.getprotobyname("icmp")
        try:
            my_socket = socket.socket(socket.AF_INET, socket.SOCK_RAW, icmp)
        except socket.error as xxx_todo_changeme:
            (errno, msg) = xxx_todo_changeme.args
            if errno == 1:
                # Operation not permitted
                msg = msg + (
                    " - Note that ICMP messages can only be sent from processes"
                    " running as root."
                )
                raise socket.error(msg)
            raise  # raise the original error
    
        my_ID = os.getpid() & 0xFFFF
    
        send_one_ping(my_socket, dest_addr, my_ID)
        delay = receive_one_ping(my_socket, my_ID, timeout)
    
        my_socket.close()
        return delay
    
    
    def verbose_ping(dest_addr, timeout=1, count=2):
        """
        Send >count< ping to >dest_addr< with the given >timeout< and display
        the result.
        """
        logging.info('Messing with : %s' % dest_addr)
        try:
            for i in xrange(count):
                logging.info('line 136')
                try:
                    delay = do_one(dest_addr, timeout)
                    logging.info('line 139' + str(delay))
                except socket.gaierror as e:
                    break
                logging.info('line 142'+str(delay))
                if delay is None:
                    pass
                else:
                    delay = delay * 1000
                    logging.info('This HO is UP : %s' % dest_addr)
                    return dest_addr
        except:
            logging.info('Error in : %s' % dest_addr)
    
    
    @app.task()
    def process_ips(items):
        logging.info('This is Items:----: %s' % items)
        up_one = verbose_ping(items)
        if up_one is not None:
            logging.info('This one is UP: %s' % up_one)
    

**my_awesome_program.py**

    
    
    from task import process_ips
    
    if __name__ == '__main__':
        for i in range(0, 256):
            for j in range(1, 256):
                ip = "192.168.%s.%s" % (str(i), str(j))
                jobs = process_ips.delay(ip)
    

**/etc/defaults/celeryd**

    
    
    # Names of nodes to start
    #   most will only start one node:
    CELERYD_NODES="worker1"
    #   but you can also start multiple and configure settings
    #   for each in CELERYD_OPTS (see `celery multi --help` for examples).
    #CELERYD_NODES="worker1 worker2 worker3"
    
    # Absolute or relative path to the 'celery' command:
    CELERY_BIN="/home/jarvis/Development/venv/bin/celery"
    #CELERY_BIN="/virtualenvs/def/bin/celery"
    # App instance to use
    # comment out this line if you don't use an app
    CELERY_APP="task"
    # or fully qualified:
    #CELERY_APP="proj.tasks:app"
    
    # Where to chdir at start.
    
    CELERYD_CHDIR="/home/jarvis/Development/pythonScrap/nmap_sub"
    
    # Extra command-line arguments to the worker
    CELERYD_OPTS="--time-limit=300 --concurrency=8 --loglevel=DEBUG"
    
    # %N will be replaced with the first part of the nodename.
    CELERYD_LOG_FILE="/var/log/celery/%N.log"
    CELERYD_PID_FILE="/var/run/celery/%N.pid"
    
    # Workers should run as an unprivileged user.
    #   You need to create this user manually (or you can choose
    #   a user/group combination that already exists, e.g. nobody).
    CELERYD_USER="jarvis"
    CELERYD_GROUP="jarvis"
    
    # If enabled pid and log directories will be created if missing,
    # and owned by the userid/group configured.
    CELERY_CREATE_DIRS=1
    


Answer: You need to set your CELERYD_USER param in settings.

Have a look here:
<http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html>

If you're using supervisor, then in your supervisor conf of celery, you'd need
to do this:

    
    
    user=<user-you-want>
    

Also, under [Example
Configuration](http://celery.readthedocs.org/en/latest/tutorials/daemonizing.html#example-
configuration) section, it's explicitly said to not run your workers as
privileged users.

