# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Snippets
#
# Some useful piece of code.

# ## Argsort
#
# Get the indices of the items in an array sorted in decreasing order (largest to smallest) _acording to the value of the items_.

# +
import numpy as np

# We create an array.
x = np.array([4, 3, 5, 1, 2])
# Argsort sort in increasing order, we use a trick: [::-1]
decreasing_indices = np.argsort(x)[::-1]
print(decreasing_indices)
print(x[decreasing_indices])
# -

# ## Defaultdict
#
# Initialise a dictionary with a default type at each potential key.

# +
from collections import defaultdict

# We create a dict where each potential key is an int
d = defaultdict(int)

# We add one to the key 'a'
d['a'] += 1
# We add five to the key 'b'
d['b'] += 5

# The dict has indeed the correct values
print(d)
# It has 0 for another key that we have never set
print(d['c'])
# -

# ## Sparse matrices

# +
import numpy as np
from scipy.sparse import csr_matrix

# Define the size of the matrix
m = 10000
n = 100

# Set some values
values = [1,
          42,
          3.14156926536,
          100,
          1000]

# Define their position in the matrix
rows = [0,
        5,
        100,
        2673,
        9999]
columns = [0,
           0,
           0,
           62,
           99]

X = csr_matrix((values, (rows, columns)), shape=(m, n))
print(X)
# -

# ## Get value with index from RDD

# +
# Create an RDD
values = sc.parallelize(['a', 'b', 'c', 'd'])

# Map each value with its index (a -> 0, b -> 1, ...)
values_with_index = values.zipWithIndex()

print(values_with_index.take(4))
# -

# ## Spark's sparse vectors 

# +
from pyspark.mllib.linalg import Vectors 

# d-dimensional vector
d = 1000

# Set indices
indices = [0, 10, 100, 999]
values = [1, 3.14, 42, 11]

# Create the sparse vector
v = Vectors.sparse(d, indices, values)

print(v)
# -

# ## Save JSON
#
# You could have a look at the `utils.py` file.

# +
from utils import save_json

dictionaries = [
    {'a': 0},
    {'b': 1},
    {'c': 2}
]

save_json(dictionaries, 'dictionaries.txt')
# -

# ## Load JSON

# +
from utils import load_json

dictionaries = load_json('dictionaries.txt')
print(dictionaries)

StarcoderdataJupyterScriptsDedupFiltered