python script speed improvements
Question: for my robot I am analyzing laser range data. I need to analyze a lot of
samples per second. So speed is required. I know python is not the right
language based on this - but I don't want to switch for now as I am in the
prototyping phase (will see if I ever get out of it :-) ).

At the moment I am stuck on squeezing more speed out of the analyzing code I
have.

I pulled out the relevant code and created a small test. It would be brilliant
if someone could give me a some hints on where to improve speed in this test
script.

    
    
    from math import degrees, radians, sin, cos, fabs
    import time
    
    class NewRobotMap(object):
        def __init__(self, sizeX, sizeY, Resolution, RobotPosX, RobotPosY, RobotTheta, ServoPos, mapMaxOcc, mapMaxFree, OccValue, EmptyValue):
            self.sizeX = sizeX
            self.sizeY = sizeY
            self.RobotPosX = int(RobotPosX)
            self.RobotPosY = int(RobotPosY)
            self.mapResolution = int(Resolution)
            self.StartPosX = int(RobotPosX)
            self.StartPosY = int(RobotPosY)
            self.RobotTheta = float(RobotTheta)
            self.EmptyValue = EmptyValue
            self.ServoPos = ServoPos
            self.mapMaxOcc = mapMaxOcc
            self.mapMaxFree = mapMaxFree
            self.mapOccValue = OccValue
            self.RobotPosOldX = ""
            self.RobotPosOldY = ""
    
        def clear(self):
            self.RobotMap = [[self.EmptyValue for i in xrange(self.sizeY)] for j in xrange(self.sizeX)]
        def updateMap(self ,x ,y , Val):
            oldval = self.RobotMap[x][y]
            self.RobotMap[x][y]=self.RobotMap[x][y] + Val
    
            if self.RobotMap[x][y] > self.mapMaxOcc:
                self.RobotMap[x][y] = self.mapMaxOcc
            elif self.RobotMap[x][y] < self.mapMaxFree:
                self.RobotMap[x][y] = self.mapMaxFree
            return oldval, self.RobotMap[x][y]
        def setOcc(self,x,y):
                self.RobotMap[x][y] = self.mapMaxOcc
        def updateRobot(self,theta,x,y):
            robotThetaold=self.RobotTheta
    
            self.RobotTheta = float(theta)
            self.RobotPosX = int(round(self.StartPosX + float(int(x)/self.mapResolution), 0))
            self.RobotPosY = int(round(self.StartPosY - float(int(y)/self.mapResolution),0))
    
            if x != self.RobotPosOldX or y != self.RobotPosOldX:
                self.RobotPosOldX = x
                self.RobotPosOldY = y
                return True
            else:
                self.RobotPosOldX = x
                self.RobotPosOldY = y
                return False
        def getRobotPos(self):
            return self.RobotPosX, self.RobotPosY
        def display(self):
            s = [[str(e) for e in row] for row in self.RobotMap]
            lens = [len(max(col, key=len)) for col in zip(*s)]
            fmt = '\t'.join('{{:{}}}'.format(x) for x in lens)
            table = [fmt.format(*row) for row in s]
            print '\n'.join(table)
        def updateServoPos(self, newServoPos):
            self.ServoPos = newServoPos
    
    
    templateData = {
        'MapWidth' : 800,
        'MapHeight': 600,
        'StartPosX' : 500,
        'StartPosY' : 300,
        'StartTheta' : 0,
        'Resolution' : 5,
        'mapThresholdFree' : 126,
        'mapThresholdOcc' : 130, #169
        'EmptyValue' : 128,
        'mapMaxOcc' : 137,
        'mapMaxFree' : 119,
        'ServoPos' : 0,
        'CurrentPosX' : 0,
        'CurrentPosY' : 0,
        'CurrentTheta' : 0,
        'SafeZone' : 10
    }
    
    templateData["MapHeight"] = templateData["MapHeight"] / templateData["Resolution"]
    templateData["MapWidth"] = templateData["MapWidth"] / templateData["Resolution"]
    templateData["StartPosX"] = templateData["StartPosX"] / templateData["Resolution"]
    templateData["StartPosY"] = templateData["StartPosY"] / templateData["Resolution"]
    
    
    def updateSonarCalcMapVal(val):
        mapThresholdFree = templateData["mapThresholdFree"]
        mapThresholdOcc = templateData["mapThresholdOcc"]
    
        #oldval
        if  val[0] <= mapThresholdFree:
            oldval = 0
        elif mapThresholdFree < val[0] < mapThresholdOcc:
            oldval = 1
        elif  val[0] >= mapThresholdOcc:
            oldval = 2
    
        # newval
        if  val[1] <= mapThresholdFree:
            newval = 0
        elif mapThresholdFree < val[1] < mapThresholdOcc:
            newval = 1
        elif  val[1] >= mapThresholdOcc:
            newval = 2
    
        if oldval != newval:
            return newval
        else:
            return 'n'
    
    
    def dur( op=None, clock=[time.time()] ):
        if op != None:
            duration = time.time() - clock[0]
            print '%s finished. Duration %.6f seconds.' % (op, duration)
        clock[0] = time.time()
    
    def updateIRWrite(RobotPos, coord, updateval):
        XtoUpdate=RobotPos[0] + coord[0]
        YtoUpdate=RobotPos[1] - coord[1]
        val = map.updateMap(XtoUpdate, YtoUpdate , updateval)
        newval=updateSonarCalcMapVal(val)
    
    ########### main Script #############
    
    map=NewRobotMap(templateData["MapWidth"],templateData["MapHeight"], templateData["Resolution"], templateData["StartPosX"],templateData["StartPosY"], templateData["StartTheta"], templateData["ServoPos"],templateData["mapMaxOcc"],templateData["mapMaxFree"],templateData["mapThresholdOcc"],templateData["EmptyValue"])
    map.clear()
    
    dur()
    for x in xrange(0,10001*40):
        updateIRWrite((100,100), (10,10), 1)
    dur("loops")
    

I tried a numpy array as self.RobotMap in the NewRobotMap class/object. But
this was much slower.


Answer: Few tips

# Minimize too deep redirections

Your code here:

    
    
    def updateMap(self ,x ,y , Val):
            oldval = self.RobotMap[x][y]
            self.RobotMap[x][y]=self.RobotMap[x][y] + Val
    
            if self.RobotMap[x][y] > self.mapMaxOcc:
                self.RobotMap[x][y] = self.mapMaxOcc
            elif self.RobotMap[x][y] < self.mapMaxFree:
                self.RobotMap[x][y] = self.mapMaxFree
            return oldval, self.RobotMap[x][y]
    

is all the time repeating `self.RobotMap[x][y]` what requires 4 levels of hops
to get the value (self -> RobotMap -> [x] -> [y])

This can be optimized:

# In place update

old:

    
    
    self.RobotMap[x][y]=self.RobotMap[x][y] + Val
    

new (saving diving for existing value second time)

    
    
    self.RobotMap[x][y] += Val
    

# Use local variable instead of deeply nested structure

    
    
    def updateMap(self ,x ,y , Val):
            oldval = self.RobotMap[x][y]
            newval = oldval + Val
    
            if newval > self.mapMaxOcc:
                newval = self.mapMaxOcc
            elif newval < self.mapMaxFree:
                newval = self.mapMaxFree
            return oldval, newval
    

Note, that your old return `oldval, self.RobotMap[x][y]` is not only returning
a value, but you have already modified the `self.RobotMap[x][y]` anyway (as it
is mutable), so if you rely on that, you could be surprised.

## Using global variables instead of tempateData dictionary

Changing dictionary into global variable speeded up the run a bit as it
removed one level ov indirection. I know, it looks nasty, but this may happen
with optimization.

# Skip returning `self.RobotMap[x][y]`

Consider saving returning `self.RobotMap[x][y]` if this not necessary, or if
you have already changed that value.

# Quick clear

change original:

    
    
    def clear(self):
        self.RobotMap = [[self.EmptyValue for i in xrange(self.sizeY)] for j in xrange(self.sizeX)]
    

to:

    
    
    def clear(self):
        self.RobotMap = self.sizeY * [self.sizeY * [self.EmptyValue]]
    

My test show about twice as fast execution for x = 3, y = 5, larger sizez
could be even better.

# Modified code - from 0.790581 to 0.479875 seconds

    
    
    from math import degrees, radians, sin, cos, fabs
    import time
    
    templ_MapWidth = 800
    templ_MapHeight = 600
    templ_StartPosX = 500
    templ_StartPosY = 300
    templ_StartTheta = 0
    templ_Resolution = 5
    templ_mapThresholdFree = 126
    templ_mapThresholdOcc = 130
    templ_EmptyValue = 128
    templ_mapMaxOcc = 137
    templ_mapMaxFree = 119
    templ_ServoPos = 0
    templ_CurrentPosX = 0
    templ_CurrentPosY = 0
    templ_CurrentTheta = 0
    templ_SafeZone = 10
    
    templ_MapHeight  = templ_MapHeight / templ_Resolution
    templ_MapWidth   = templ_MapWidth /  templ_Resolution
    templ_StartPosX  = templ_StartPosX / templ_Resolution
    templ_StartPosY  = templ_StartPosY / templ_Resolution
    
    class NewRobotMap(object):
        def __init__(self, sizeX, sizeY, Resolution, RobotPosX, RobotPosY, RobotTheta, ServoPos, mapMaxOcc, mapMaxFree, OccValue, EmptyValue):
            self.sizeX = sizeX
            self.sizeY = sizeY
            self.RobotPosX = int(RobotPosX)
            self.RobotPosY = int(RobotPosY)
            self.mapResolution = int(Resolution)
            self.StartPosX = int(RobotPosX)
            self.StartPosY = int(RobotPosY)
            self.RobotTheta = float(RobotTheta)
            self.EmptyValue = EmptyValue
            self.ServoPos = ServoPos
            self.mapMaxOcc = mapMaxOcc
            self.mapMaxFree = mapMaxFree
            self.mapOccValue = OccValue
            self.RobotPosOldX = ""
            self.RobotPosOldY = ""
    
        def clear(self):
            self.RobotMap = self.sizeX * [self.sizeY * [self.EmptyValue]]
    
        def updateMap(self, x, y, Val):
            oldval = self.RobotMap[x][y]
            newval = oldval + Val
    
            if newval < self.mapMaxFree:
                return oldval, self.mapMaxFree
            if newval > self.mapMaxOcc:
                return oldval, self.mapMaxOcc
            return oldval, newval
    
        def setOcc(self, x, y):
                self.RobotMap[x][y] = self.mapMaxOcc
    
        def updateRobot(self, theta, x, y):
            robotThetaold = self.RobotTheta
    
            self.RobotTheta = float(theta)
            self.RobotPosX = int(round(self.StartPosX + float(int(x)/self.mapResolution), 0))
            self.RobotPosY = int(round(self.StartPosY - float(int(y)/self.mapResolution), 0))
    
            if x != self.RobotPosOldX or y != self.RobotPosOldX:
                self.RobotPosOldX = x
                self.RobotPosOldY = y
                return True
            else:
                self.RobotPosOldX = x
                self.RobotPosOldY = y
                return False
    
        def getRobotPos(self):
            return self.RobotPosX, self.RobotPosY
    
        def display(self):
            s = [[str(e) for e in row] for row in self.RobotMap]
            lens = [len(max(col, key=len)) for col in zip(*s)]
            fmt = '\t'.join('{{:{}}}'.format(x) for x in lens)
            table = [fmt.format(*row) for row in s]
            print '\n'.join(table)
    
        def updateServoPos(self, newServoPos):
            self.ServoPos = newServoPos
    
    def updateSonarCalcMapVal(org, new):
        mapThresholdFree = templ_mapThresholdFree
        mapThresholdOcc = templ_mapThresholdOcc
    
        #oldval
        if  org <= mapThresholdFree:
            oldval = 0
        elif mapThresholdFree < org < mapThresholdOcc:
            oldval = 1
        elif  org >= mapThresholdOcc:
            oldval = 2
    
        # newval
        if  new <= mapThresholdFree:
            newval = 0
        elif mapThresholdFree < new < mapThresholdOcc:
            newval = 1
        elif  new >= mapThresholdOcc:
            newval = 2
    
        if oldval != newval:
            return newval
        else:
            return 'n'
    
    
    def dur(op=None, clock=[time.time()]):
        if op != None:
            duration = time.time() - clock[0]
            print '%s finished. Duration %.6f seconds.' % (op, duration)
        clock[0] = time.time()
    
    def updateIRWrite(RobotPos, coord, updateval):
        XtoUpdate = RobotPos[0] + coord[0]
        YtoUpdate = RobotPos[1] - coord[1]
    
        newval = updateSonarCalcMapVal(*mymap.updateMap(XtoUpdate, YtoUpdate, updateval))
    
    ########### main Script #############
    
    mymap = NewRobotMap(templ_MapWidth, templ_MapHeight, templ_Resolution, templ_StartPosX, templ_StartPosY, templ_StartTheta, templ_ServoPos, templ_mapMaxOcc, templ_mapMaxFree, templ_mapThresholdOcc, templ_EmptyValue)
    mymap.clear()
    
    dur()
    
    for x in xrange(0, 10001*40):
        updateIRWrite((100, 100), (10, 10), 1)
    
    dur("loops")
    

# Conclusions

The code definitely needs review for doing correct work. E.g. there are
methods, which are not used at all and other calls, which never use returned
value.

But some optimization could be shown. Generally, following is good to follow:

  1. Make your code running correctly first
  2. Clarify what is acceptable speed, do not optimize, if not necessary
  3. Measure, profile
  4. Start optimizing in busiest loops, there are best chances to speed things up. In them, each line of code counts.

