# Serializing the code that runs on the executors

`>>> rdd = sc.parallelize([1, 3], numSlices=2)>>> import math>>> def sqrt(a):…     return math.sqrt(a)…>>> def pyth_add(a,b):…     return sqrt(a ** 2 + b ** 2)…>>> rdd.reduce(lambda x,y: pyth_add(x,y))3.1622776601683795`
`>>> import numpy as np>>> rdd = sc.parallelize([np.array([1,2,3]), np.array([1,2,3])], numSlices=2)>>> rdd.reduce(lambda x,y: np.dot(x,y))[Stage 0:> (0 + 2) / 2]18/11/09 18:29:17 WARN scheduler.TaskSetManager: Lost task 0.0 in stage 0.0 (TID 0, xx-xx-xx-xx-xx-xx.xxx.xxx.criteo.prod, executor 1): org.apache.spark.api.python.PythonException: Traceback (most recent call last):...ModuleNotFoundError: No module named 'numpy'`

# Using PEX to ship code

`\$ pex numpy -o myarchive.pex\$ export PYSPARK_DRIVER_PYTHON=`which python`\$ export PYSPARK_PYTHON=./myarchive.pex\$ pyspark \--conf spark.executorEnv.PEX_ROOT=./.pex \--master yarn \--deploy-mode client \--files myarchive.pex>>> import numpy as np>>> rdd = sc.parallelize([np.array([1,2,3]), np.array([1,2,3])], numSlices=2)>>> rdd.reduce(lambda x,y: np.dot(x,y))14`

# Developing PySpark applications

`\$ pip list --exclude-editable --format json[{"name": "numpy", "version": "1.15.4"}, {"name": "pex", "version": "1.5.2"}, {"name": "pip", "version": "10.0.1"}, {"name": "setuptools", "version": "39.0.1"}, {"name": "wheel", "version": "0.31.1"}]`
`\$ pip list -e --format json[{"name": "userlib", "version": "0.0.1"}]`

# Running in Prod

`\$ pex pyspark==2.3.2 numpy userlib -o myarchive.pex                 \$./myarchive.pex -m userlib.startup  14`

# Wrap-up

Written by

Written by

## Fabian Höring

#### Software Engineer @ Criteo
{"ft_lang_label":"__label__en","ft_lang_prob":0.90042067,"math_prob":0.5839018,"size":12337,"snap":"2020-34-2020-40","text_gpt3_token_len":2657,"char_repetition_ratio":0.13054407,"word_repetition_ratio":0.024169184,"special_character_ratio":0.21309881,"punctuation_ratio":0.105240546,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.9640215,"pos_list":[0,1,2],"im_url_duplicate_count":[null,1,null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2020-08-06T20:39:29Z\",\"WARC-Record-ID\":\"<urn:uuid:b6d6f3f6-56f9-4d64-9f5b-e20b0e566134>\",\"Content-Length\":\"243042\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:0b58587f-2ced-4681-98fe-2db4db5fd08f>\",\"WARC-Concurrent-To\":\"<urn:uuid:edcc61df-72de-44f7-883d-8c23e93f5d4a>\",\"WARC-IP-Address\":\"104.16.121.127\",\"WARC-Target-URI\":\"https://medium.com/criteo-labs/packaging-code-with-pex-a-pyspark-example-9057f9f144f3\",\"WARC-Payload-Digest\":\"sha1:O2KPBSX3PL6US34KF5JBZVEYHWLREXOP\",\"WARC-Block-Digest\":\"sha1:Z3RHJXUZFPIBKO3PJKDOS33OAFNWXIID\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2020/CC-MAIN-2020-34/CC-MAIN-2020-34_segments_1596439737019.4_warc_CC-MAIN-20200806180859-20200806210859-00206.warc.gz\"}"}