
#splunk
SPLUNK = {
    'host': 'MY.SPLUNK.com',
    'scheme': 'https',
    'port': 8089,
    'username': 'MY_SPLUNK_USER',
    'password': 'MY_SPLUNK_PWD'   
}


"""
Explanation: Splunk<> Graphistry
Graphistry brings modern visual analytics to event data in Splunk. The full platform is intended for enterprise teams, while this tutorials shares visibility techniques for researchers and hunters.
To use:
* Read along, start the prebuilt visualizations by clicking on them
* Plug in your Graphistry API Key & Splunk credentials to use for yourself
Further reading:
* UI Guide: https://labs.graphistry.com/graphistry/ui.html
* Python client tutorials & demos: https://github.com/graphistry/pygraphistry 
* Graphistry API Key: https://www.graphistry.com/api-request 
* DoD / VAST challenges: https://www.cs.umd.edu/hcil/varepository/benchmarks.php 
0. Configure
End of explanation
"""


import pandas as pd

"""
Explanation: 1. Imports
End of explanation
"""


!pip install graphistry

import graphistry
graphistry.__version__

# To specify Graphistry account & server, use:
# graphistry.register(api=3, username='...', password='...', protocol='https', server='hub.graphistry.com')
# For more options, see https://github.com/graphistry/pygraphistry#configure

"""
Explanation: Graphistry
End of explanation
"""


!pip install splunk-sdk

import splunklib

#Connect to Splunk. Replace settings with your own setup.
import splunklib.client as client
import splunklib.results as results

service = client.connect(**SPLUNK)

def extend(o, override):
  for k in override.keys():
    o[k] = override[k]
  return o

STEP = 10000;                       
def splunkToPandas(qry, overrides={}):
    kwargs_blockingsearch = extend({
        "count": 0,
        "earliest_time": "2010-01-24T07:20:38.000-05:00",
        "latest_time": "now",
        "search_mode": "normal",
        "exec_mode": "blocking"
    }, overrides)
    job = service.jobs.create(qry, **kwargs_blockingsearch)

    print "Search results:\n"
    resultCount = job["resultCount"]
    offset = 0;                         

    print 'results', resultCount
    out = None
    while (offset < int(resultCount)):
        print "fetching:", offset, '-', offset + STEP
        kwargs_paginate = extend(kwargs_blockingsearch,
                                 {"count": STEP,
                                  "offset": offset})

        # Get the search results and display them
        blocksearch_results = job.results(**kwargs_paginate)
        reader = results.ResultsReader(blocksearch_results)
        lst = [x for x in reader]
        df2 = pd.DataFrame(lst)    
        out = df2 if type(out) == type(None) else pd.concat([out, df2], ignore_index=True)
        offset += STEP
    return out        

"""
Explanation: Splunk
End of explanation
"""


query = 'search index="vast" srcip=* destip=* | rename destip -> dest_ip, srcip -> src_ip | fields dest_ip _time src_ip protocol | eval time=_time | fields - _* '
%time df = splunkToPandas(query, {"sample_ratio": 1000})

#df = splunkToPandasAll('search index="vast" | head 10')
#df = pd.concat([ splunkToPandas('search index="vast" | head 10'), splunkToPandas('search index="vast" | head 10') ], ignore_index=True)


print 'results', len(df)

df.sample(5)

"""
Explanation: 2. Get data
End of explanation
"""


graphistry.bind(source='src_ip', destination='dest_ip').edges(df).plot()

"""
Explanation: 3. Visualize!
A) Simple IP<>IP: 1326 nodes, 253K edges
End of explanation
"""


def make_edges(df, src, dst):
  out = df.copy()
  out['src'] = df[src]
  out['dst'] = df[dst]
  return out



ip2ip = make_edges(df, 'src_ip', 'dest_ip')
srcip2protocol = make_edges(df, 'src_ip', 'protocol')

combined = pd.concat([ip2ip, srcip2protocol], ignore_index=True)
combined.sample(6)

graphistry.bind(source='src', destination='dst').edges(combined).plot()

"""
Explanation: B) IP<>IP + srcip<>protocol: 1328 nodes, 506K edges
End of explanation
"""


hg = graphistry.hypergraph(df, entity_types=[ 'src_ip', 'dest_ip', 'protocol'] )
print hg.keys()
hg['graph'].plot()

"""
Explanation: 3. All<>All via Hypergraph: 254K nodes, 760K edges
End of explanation
"""



nodes = pd.concat([ 
    df[['src_ip']].rename(columns={'src_ip': 'id'}).assign(orig_col='src_ip'), 
    df[['dest_ip']].rename(columns={'dest_ip': 'id'}).assign(orig_col='dest_ip') ], 
    ignore_index=True).drop_duplicates(['id'])

#see https://labs.graphistry.com/docs/docs/palette.html
col2color = { 
    "src_ip": 90005,
    "dest_ip": 46005   
}

nodes_with_color = nodes.assign(color=nodes.apply(lambda row: col2color[ row['orig_col'] ], axis=1))

nodes_with_color.sample(3)

graphistry.bind(source='src_ip', destination='dest_ip').edges(df).nodes(nodes_with_color).bind(node='id', point_color='color').plot()

"""
Explanation: Node Colors
End of explanation
"""


