# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

# install pyvis framework for visualization
print("Installing pyvis:")
!pip install pyvis

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import pandas as pd
import numpy as np
import networkx as nx
from pyvis.network import Network

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
# Python Libraries as a usage-clustered network

This notebook is trying to visualize the major existing python libraries as a clustored network. This work has **no claim regarding completeness**. Nevertheless, I appreciate any ideas on adding missing packages or improving clean code that would make the network maintenance much easier.
"""
"""
## Defining nodes

First, we have to add the nodes that will in general present the differen libraries. Furthermore, there are main- and subnodes which can be treated as the clusters headline.

The network graph itself basically follows a tree structure where **"Python"** is assigned to the **root node**. Edges that are leaving the root node will lead to **topical main clusters** that describe the different areas python is used in. Following that idea, one can find the same approach leaving the main clusters and leading to **topical sub clusters** which are more granular. Consequently, the deepest layer will be defined by the **specific libraries**.

With that setting in mind, it is clear that in some cases a library might be a *leaf node on the bottom* and a *direct child from another main cluster* at the same time. This observation makes it **hard to balance the network graph as a tree**.

Let's start defining the node clusters:
"""
root = ["Python"]
main_cluster = [
    "Visualization", 
    "Machine Learning", 
    "NLP", 
    "Testing", 
    "Data Analysis / EDA", 
    "Data Processing", 
    "Finance",
    "Basic Usage",
]
sub_cluster = [
    "Dashboard",
    "D3.js",
    "Extraordinary Visualization", 
    "Feature Engineering", 
    "FlowChart",
    "Geo",
    "Graph / Network",
    "Hyperparameter Tuning", 
    "Information Retrieval", 
    "Integration", 
    "Interactive Visualization",
    "Keras",
    "Performance",
    "Pytorch",
    "Static Visualization",
    "TimeSeries",
    "Tensorflow",
    "TPOT",
    "XAI",

]
libraries = [
     'altair',
     'auto-sklearn',
     'auto-viml',
     'bokeh',
     'candela',
     'cufflinks',
     'd3py',
     'dash',
     'dask',
     'dtale',
     'dtreeviz',
     'dx_analytics',
     'eli5',
     'featuretools',
     'folium',
     'functools',
     'fundamentalanalysis',
     'gensim',
     'geopandas',
     'geopy',
     'graph-tool',
     'holoview',
     'itertools',
     'jaal',
     'lazypredict',
     'leaflet',
     'lime',
     'luminaire',
     'lux',
     'matplotlib',
     'missingno',
     'more-itertools',
     'mutmut',
     'neptune',
     'networkx',
     'neuraxle',
     'nltk',
     'numpy',
     'nvd3',
     'pandas',
     'pandas-datareader',
     'pandas-log',
     'pandas-profiler',
     'pandera',
     'panel',
     'pattern',
     'plotly',
     'polyglot',
     'pycaret',
     'pydantic',
     'pyflowchart',
     'pynlpl',
     'pypolars',
     'pystemmer',
     'pyterrier',
     'pytest',
     'pyts',
     'pyvis',
     'roughviz',
     'rshap',
     'scikit-learn',
     'scikit-optimize',
     'seaborn',
     'shapash',
     'sigma.js',
     'sklearn laboratory',
     'sklearn-deap',
     'sklearn-pandas',
     'sklearn-xarray',
     'spacy',
     'sql-test',
     'stanford corenlp python',
     'stumpy',
     'sweetviz',
     'textblob',
     'tpot',
     'tsfresh',
     'vaex',
     'visdcc',
     'vocabulary',
     'yfinance'
]
"""
## Defininng the edges

Next we define the basic structur for our edges:
"""
# set up the edges that are building the graph
data = [
    ["Python", "Basic Usage"],
    ["Python", "Finance"],
    ["Python", "Data Processing"],
    ["Python", "Data Analysis / EDA"],
    ["Python", "Machine Learning"],
    ["Python", "Visualization"],
    ["Python", "Testing"],
    
    ["Basic Usage","functools"],
    ["Basic Usage", "itertools"],
    ["Basic Usage", "more-itertools"],
    
    ["Data Processing", "pandas"],
    ["Data Processing", "numpy"],
    ["Data Processing", "Performance"],
    
    ["Performance", "vaex"],
    ["Performance", "pypolars"],
    ["Performance", "dask"],
    
    ["Finance", "cufflinks"],
    ["Finance", "dx_analytics"],
    ["Finance", "fundamentalanalysis"],
    ["Finance", "yfinance"],
    
    ["cufflinks", "Visualization"],
    
    ["Machine Learning", "Feature Engineering"],
    ["Machine Learning", "Hyperparameter Tuning"],
    ["Machine Learning", "Keras"],
    ["Machine Learning", "NLP"],
    ["Machine Learning", "Pytorch"],
    ["Machine Learning", "Tensorflow"],
    ["Machine Learning", "TPOT"],
    ["Machine Learning", "XAI"],
    ["Machine Learning", "auto-viml"],
    ["Machine Learning", "dtreeviz"], 
    ["Machine Learning", "pycaret"],
    ["Machine Learning", "lazypredict"],
    ["Machine Learning", "scikit-learn"],
    
    ["pycaret", "lazypredict"],
    
    ["Feature Engineering", "featuretools"],
    ["Feature Engineering", "tsfresh"],
    
    ["scikit-learn", "sklearn-xarray"],
    ["scikit-learn", "sklearn-pandas"],
    ["scikit-learn", "sklearn-deap"],
    ["scikit-learn", "auto-sklearn"],
    ["scikit-learn", "sklearn laboratory"],
    
    ["sklearn laboratory", "neptune"],
    
    ["Hyperparameter Tuning", "neuraxle"],
    ["Hyperparameter Tuning", "scikit-optimize"],
    
    ["XAI", "auto-viml"],
    ["XAI", "eli5"],
    ["XAI", "rshap"],
    ["XAI", "lime"],
    
    ["NLP", "Information Retrieval"],
    ["NLP", "nltk"],
    ["NLP", "gensim"],
    ["NLP", "scikit-learn"],
    ["NLP", "spacy"],
    ["NLP", "pystemmer"],
    ["NLP", "stanford corenlp python"],
    ["NLP", "textblob"],
    ["NLP", "pattern"],
    ["NLP", "polyglot"],
    ["NLP", "pynlpl"],
    ["NLP", "vocabulary"],
    
    ["Information Retrieval", "pyterrier"],
    
    ["Visualization", "D3.js"],
    ["Visualization", "Dashboard"],
    ["Visualization", "Extraordinary Visualization"],
    ["Visualization", "FlowChart"],
    ["Visualization", "Geo"],
    ["Visualization", "Graph / Network"],
    ["Visualization", "Interactive Visualization"],
    ["Visualization", "Static Visualization"],
    
    ["D3.js", "d3py"],
    ["D3.js", "nvd3"],
    
    ["Dashboard", "bokeh"],
    ["Dashboard", "dash"],
    ["Dashboard", "holoview"],
    ["Dashboard", "panel"],
    ["Dashboard", "plotly"],
    
    ["Extraordinary Visualization", "candela"],
    ["Extraordinary Visualization", "roughviz"],
    
    ["FlowChart", "pyflowchart"],
    
    ["Geo", "geopy"],
    ["Geo", "folium"],
    ["Geo", "leaflet"],
    ["Geo", "geopandas"],
    
    ["Graph / Network", "dtreeviz"],
    ["Graph / Network", "graph-tool"],
    ["Graph / Network", "jaal"],
    ["Graph / Network", "networkx"],
    ["Graph / Network", "pyvis"],
    ["Graph / Network", "sigma.js"],
    ["Graph / Network", "visdcc"],
    
    ["Interactive Visualization", "altair"],
    ["Interactive Visualization", "bokeh"],
    ["Interactive Visualization", "cufflinks"],
    ["Interactive Visualization", "holoview"],
    ["Interactive Visualization", "plotly"],
    ["Interactive Visualization", "shapash"],

    ["Static Visualization", "lux"],
    ["Static Visualization", "matplotlib"],
    ["Static Visualization", "missingno"],
    ["Static Visualization", "pandas-profiler"],
    ["Static Visualization", "seaborn"],
    ["Static Visualization", "shapash"],
    
    ["Data Analysis / EDA", "TimeSeries"],
    ["Data Analysis / EDA", "dtale"],
    ["Data Analysis / EDA", "lux"],
    ["Data Analysis / EDA", "shapash"],
    ["Data Analysis / EDA", "sweetviz"],
    ["Data Analysis / EDA", "missingno"],
    ["Data Analysis / EDA", "pandas"],
    
    ["pandas", "pandas-datareader"],
    ["pandas", "pandas-log"],
    ["pandas", "pandas-profiler"],
    
    ["TimeSeries", "luminaire"],
    ["TimeSeries", "pyts"],
    ["pandas", "sklearn-pandas"],

    ["TimeSeries", "stumpy"],

    ["visdcc", "dash"],

    ["Testing", "Integration"],
    ["Testing", "mutmut"],
    ["Testing", "pytest"],
    ["Testing", "sql-test"],

    ["Integration", "pandera"],
    ["Integration", "pydantic"],
]

"""
## Building the network graph

Next, we will add both, nodes and edges, to our graph. It is important to recalculate which id or number a node is assigned to, in order to draw the edges correctly. 
"""
"""Define edges."""
from more_itertools import locate

test_nw = Network(height='750px', width="100%", notebook=True)
nodes = root + main_cluster + sub_cluster + libraries

# add root node
root_node = list(locate(nodes, lambda x: x in root))
root_size, root_color = [35 for _ in root], ["red" for _ in root]

test_nw.add_nodes(root_node, size=root_size, label=root, color=root_color)

# add main_cluster nodes
main_nodes = list(locate(nodes, lambda x: x in main_cluster))
main_size, main_color = [30 for _ in main_cluster], ["orange" for _ in main_cluster]

test_nw.add_nodes(main_nodes, size=main_size, label=main_cluster, color=main_color)

# add sub_cluster nodes
sub_nodes = list(locate(nodes, lambda x: x in sub_cluster))
sub_size, sub_color = [25 for _ in sub_cluster], ["yellow" for _ in sub_cluster]

test_nw.add_nodes(sub_nodes, size=sub_size, label=sub_cluster, color=sub_color)

# add library nodes
lib_nodes = list(locate(nodes, lambda x: x in libraries))
lib_size, lib_color = [15 for _ in libraries], ["blue" for _ in libraries]

test_nw.add_nodes(lib_nodes, size=lib_size, label=libraries, color=lib_color)

# add edges
for edge in data:
    node_from, node_to = list(locate(nodes, lambda x: x in edge))
    test_nw.add_edge(node_from, node_to)

test_nw.show("test.html")