# Optimising Probabilistic Weighted Moving Average (PEWMA) df.iterrows loop in Pandas

I'm implementing the Probabilistic Exponentially Weighted Mean for real time prediction of sensor data in pandas but have issues with optimising the pandas notebook for quick iterations.

Is there a more optimal way to completely remove the for loop as it currently runs longer than expected. How can I take advantage of apply() ; or vectorized operations etc. here?

import pandas as pd
import numpy as np
from datetime import datetime, timedelta
import datetime
import matplotlib.pyplot as plt
#plt.style.use('fivethirtyeight')
#%config InlineBackend.figure_format = 'retina'
#%matplotlib inline
from itertools import islice
from math import sqrt
from scipy.stats import norm

ts = pd.date_range(start ='1-1-2019',
end ='1-10-2019', freq ='5T')

np.random.seed(seed=1111)
data = np.random.normal(2.012547, 1.557331,size=len(ts))
df = pd.DataFrame({'timestamp': ts, 'speed': data})
df.speed = df.speed.abs()
df = df.set_index('timestamp')
time_col = 'timestamp'
value_col = 'speed'

#pewna parameters
T = 30      # initialization period (in cycles)
beta = 0.5  # lower values make the algorithm behave more like regular EWMA
a = 0.99    # the maximum value of the EWMA a parameter, used for outliers
z = 3

#the PEWNA Model

# create a DataFrame for the run time variables we'll need to calculate
pewm = pd.DataFrame(index=df.index, columns=['Mean', 'Var', 'Std'], dtype=float)
pewm.iloc = [df.iloc[value_col], 0, 0]

t = 0

for _, row in islice(df.iterrows(), 1, None):
diff = row[value_col] - pewm.iloc[t].Mean # difference from moving average
p = norm.pdf(diff / pewm.iloc[t].Std) if pewm.iloc[t].Std != 0 else 0 # Prob of observing diff
a_t = a * (1 - beta * p) if t > T else 1 - 1/(t+1) # weight to give to this point
incr = (1 - a_t) * diff

# Update Mean, Var, Std
pewm.iloc[t+1].Mean = pewm.iloc[t].Mean + incr
pewm.iloc[t+1].Var = a_t * (pewm.iloc[t].Var + diff * incr)
pewm.iloc[t+1].Std = sqrt(pewm.iloc[t+1].Var)
t += 1


• @Graipher Ive added dummy data to show what the df looks like. – Matimba Jun 10 '19 at 9:24

I think you can use just raw numpy arrays for speeding it up:

after #the PEWMA Model you can just use following code:

_x = df[value_col]
_mean, _std, _var = np.zeros(_x.shape), np.zeros(_x.shape), np.zeros(_x.shape)

for i in range(1, len(_x)):
diff = _x[i] - _mean[i-1]

p = norm.pdf(diff / _std[i-1]) if _std[i-1] != 0 else 0 # Prob of observing diff
a_t = a * (1 - beta * p) if (i-1) > T else 1 - 1/i # weight to give to this point
incr = (1 - a_t) * diff

# Update Mean, Var, Std
v = a_t * (_var[i-1] + diff * incr)
_mean[i] = _mean[i-1] + incr
_var[i] = v
_std[i] = np.sqrt(v)

pewm = pd.DataFrame({'Mean': _mean, 'Var': _var, 'Std': _std}, index=df.index)


If you need to apply it to really big data you could make it even faster by using numba package:

from numba import njit

# numba has some issues with stats's norm.pdf so redefine it here as function
@njit
def norm_pdf(x):
return np.exp(-x**2/2)/np.sqrt(2*np.pi)

@njit
def pwma(_x, a, beta, T):
_mean, _std, _var = np.zeros(_x.shape), np.zeros(_x.shape), np.zeros(_x.shape)
_mean = _x

for i in range(1, len(_x)):
diff = _x[i] - _mean[i-1]

p = norm_pdf(diff / _std[i-1]) if _std[i-1] != 0 else 0 # Prob of observing diff
a_t = a * (1 - beta * p) if (i-1) > T else 1 - 1/(i) # weight to give to this point
incr = (1 - a_t) * diff

# Update Mean, Var, Std
v = a_t * (_var[i-1] + diff * incr)
_mean[i] = _mean[i-1] + incr
_var[i] = v
_std[i] = np.sqrt(v)
return _mean, _var, _std

# Using :
_mean, _var, _std = pwma(df[value_col].values, 0.99, 0.5, 30)
pewm = pd.DataFrame({'Mean': _mean, 'Var': _var, 'Std': _std}, index=df.index)
{"ft_lang_label":"__label__en","ft_lang_prob":0.5215165,"math_prob":0.9925167,"size":1919,"snap":"2021-04-2021-17","text_gpt3_token_len":565,"char_repetition_ratio":0.109138384,"word_repetition_ratio":0.0,"special_character_ratio":0.31109953,"punctuation_ratio":0.17632242,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.99987674,"pos_list":[0],"im_url_duplicate_count":[null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2021-01-16T12:09:27Z\",\"WARC-Record-ID\":\"<urn:uuid:a13684ea-f830-49cb-8468-a8d5952837e3>\",\"Content-Length\":\"154559\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:60c668ac-a061-482e-9049-890da02dbba9>\",\"WARC-Concurrent-To\":\"<urn:uuid:e97db4a4-a138-4993-86d0-c4ccffe4d46e>\",\"WARC-IP-Address\":\"151.101.65.69\",\"WARC-Target-URI\":\"https://codereview.stackexchange.com/questions/221962/optimising-probabilistic-weighted-moving-average-pewma-df-iterrows-loop-in-pan\",\"WARC-Payload-Digest\":\"sha1:2NTMX4B2KKFD5XQIUHDDOGG5QE54ODUI\",\"WARC-Block-Digest\":\"sha1:SZXJIWZ3OSBNC5MEMZX62RJ6W5YQDDHQ\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2021/CC-MAIN-2021-04/CC-MAIN-2021-04_segments_1610703506640.22_warc_CC-MAIN-20210116104719-20210116134719-00174.warc.gz\"}"}