#graphistry
# To specify Graphistry account & server, use:
# graphistry.register(api=3, username='...', password='...', protocol='https', server='hub.graphistry.com')
# For more options, see https://github.com/graphistry/pygraphistry#configure  

#splunk
SPLUNK = {
    'host': 'SPLUNK.MYSITE.COM',
    'scheme': 'https',
    'port': 8089,
    'username': 'corelight_tutorial',
    'password': 'MY_SPLUNK_PWD'   
}

"""
Explanation: Threat Hunting Masterclass: Three data science notebooks for finding bad actors in your network logs
More info: https://www.graphistry.com/blog/zeek-masterclass
To get start,load logs.tar into your Splunk and replace the index used below, or call it index corelight_tutorial: https://data.world/graphistry/networkforensics . From there, follow the cells below.
Purpose
These tutorials cover multiple useful topic areas:

Hunts: Sample queries & visualizations for looking at encrypted traffic, DNS tunneling, network shares & logins, & file obfuscation. 
Data types: Network logs around TLS, DNS, NTLM, SMB, and more
Methodologies: Data science notebooks, SIEM queries, visual graph analytics
Tools: Jupyter, Splunk, Bro/Zeek/Corelight, and Graphistry

Configure

If you are using Graphistry Marketplace, leave GRAPHISTRY unedited, else, uncomment and fill it in
Fill in SPLUNK. Make sure the user has capabilities for REST API access and reading the index in which you put logs.tar
End of explanation
"""


!pip install graphistry -q
!pip install splunk-sdk -q

import pandas as pd
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

import sys
import numpy as np
import math
np.set_printoptions(threshold=sys.maxsize)

import re

import graphistry
graphistry.register(**GRAPHISTRY)

import splunklib
import splunklib.client as client
import splunklib.results as results

service = client.connect(**SPLUNK)

"""
Explanation: Imports
End of explanation
"""


def safe_log(v):
  try:
     v2 = float(v)
     return math.log(round(v2) + 1) if not np.isnan(v2) else 0
  except:
    return 0
  
  
# Convert bytes to log of numbers
# Running this twice is safe (idempotent)
# Returns a copy (no mutation of the original)
def log_of_bytes(df):
  df2 = df.copy()
  for c in [c for c in df.columns if re.match('.*bytes.*', c) and not re.match('log\(.*', c)]:
    df2['log(' + c + ')'] = df[c].apply(safe_log)      
  return df2

"""
Explanation: Helpers
General
End of explanation
"""


STEP = 50000;                       
def splunkToPandas(qry, overrides={}):
    kwargs_blockingsearch = {
        "count": 0,
        "earliest_time": "2010-01-24T07:20:38.000-05:00",
        "latest_time": "now",
        "search_mode": "normal",
        "exec_mode": "blocking",
        **overrides}
    job = service.jobs.create(qry, **kwargs_blockingsearch)

    print("Search results:\n")
    resultCount = job["resultCount"]
    offset = 0;                         

    print('results', resultCount)
    out = None
    while (offset < int(resultCount)):
        print("fetching:", offset, '-', offset + STEP)
        kwargs_paginate = {**kwargs_blockingsearch,
                           "count": STEP,
                           "offset": offset}

        # Get the search results and display them
        blocksearch_results = job.results(**kwargs_paginate)
        reader = results.ResultsReader(blocksearch_results)
        lst = [x for x in reader]
        df2 = pd.DataFrame(lst)    
        out = df2 if type(out) == type(None) else pd.concat([out, df2], ignore_index=True)
        offset += STEP
    for c in out.columns:
        out[c] = out[c].astype(str)
    return out

"""
Explanation: Splunk

Query splunk, with optional args like sampleRate
Automatically paginate when result split over multiple responses
Return as a Pandas dataframe (Note: treats all cols as strings)
End of explanation
"""


categories = {
    'ip': ['id.orig_h', 'id.resp_h']
}

opts={
    'CATEGORIES': categories 
}

"""
Explanation: Bro/Zeek
Useful bindings for hypergraphs
End of explanation
"""


##Extend graphistry.plotter.Plotter to add chainable method "my+graph.color_points_by('some_column_name')..." (and "color_edges_by")

import graphistry.plotter

def color_col_by_categorical(df, type_col):
  types = list(df[type_col].unique())
  type_to_color = {t: i for (i, t) in enumerate(types)}
  return df[type_col].apply(lambda t: type_to_color[t])

def color_col_by_continuous(df, type_col):
  mn = df[type_col].astype(float).min()
  mx = df[type_col].astype(float).max()
  if mx - mn < 0.000001:
    print('warning: too small values for color_col_by_continuous')
    return color_col_by_categorical(df, type_col)
  else:
    print('coloring for range', mn, mx)
  return df[type_col].apply(lambda v: 228010 - round(10 * (float(v) - mn)/(mx - mn) ))
  

## g * str * 'categorical' | 'continuous' -> g
def color_points_by(g, type_col, kind='categorical'):
  fn = color_col_by_categorical if kind == 'categorical' else color_col_by_continuous
  colors = fn(g._nodes, type_col)
  return g.nodes( g._nodes.assign(point_color=colors) ).bind(point_color='point_color')

## g * str * 'categorical' | 'continuous' -> g
def color_edges_by(g, type_col, kind='categorical'):
  fn = color_col_by_categorical if kind == 'categorical' else color_col_by_continuous
  colors = fn(g._edges, type_col)
  return g.edges( g._edges.assign(edge_color=colors) ).bind(edge_color='edge_color')

graphistry.plotter.Plotter.color_points_by = color_points_by
graphistry.plotter.Plotter.color_edges_by = color_edges_by

## remove node/edges pointing to "*::nan" values
def safe_not_nan(prog, v):
  try: 
    return not prog.match(v)
  except:
    return True
  
def drop_nan_col(df, col, prog):
  not_nans = df[col].apply(lambda v: safe_not_nan(prog, v))
  return df[ not_nans == True ]
  
def drop_nan(g, edges = ['src', 'dst'], nodes = ['nodeID']):
  prog = re.compile(".*::nan$")
  edges2 = g._edges
  for col_name in g._edges.columns:
    edges2 = drop_nan_col(edges2, col_name, prog)
  nodes2 = g._nodes
  for col_name in g._nodes.columns:
    nodes2 = drop_nan_col(nodes2, col_name, prog)
  return g.nodes(nodes2).edges(edges2)
  
graphistry.plotter.Plotter.drop_hyper_nans = drop_nan  

"""
Explanation: Graphistry
End of explanation
"""


df = splunkToPandas(
    """
    search index=corelight_tutorial 
    | dedup id.orig_h, id.resp_h, name 
    | fields - _* 
    | head 100
    """,
    {'sample_ratio': 10}) # Optional, means "sample 1 in 10"

print('# rows', len(df))
df.sample(3)

hg = graphistry.hypergraph(
    df, 
    ["id.orig_h", "id.resp_h", "name", "uid"], 
    direct=True,
    opts={
        'CATEGORIES': {
            'ip': ['id.orig_h', 'id.resp_h'] # combine repeats across columns into the same nodes
        }
    })
hg['graph'].plot()

"""
Explanation: Notebook intro:
What are notebooks & why
Notebooks and their code ecosystem does a few things at the technical level:
* Web-based UI that exposes a paired Python shell session: a super terminal
* Write code, run it, see results, try again, and save your session
* Quickly connect to databases and wrangle data using the pydata Python ecosystem
Top and big teams are adopting notebook environments like Jupyter to solve some key problems:
* Individual advanced individuals use them for accessing the increasingly dominant Python ecosystem
  * Fast: Use at the beginning of a project for rapid analysis & rapid prototyping
  * Smart: Easiest way to use most machine learning tools
* Teams use them as a way to collaborate: 
  * Share executable investigations for one-offs
  * Lightweight automation:  investigation plays & rule/model analyses
  * Training
Jupyter

Edit and run a code cell and see it's output: shift-enter or via the UI
You can always edit it and rerun
Best practice: Write in order as if a full program, so you can always restart and run from th top

Google Colab

Hit Connect on the top-right to start a running personal session  for this -- it is ready when it says Connected. 
Run each cell of the notebook in sequence: either press the play button to the left of the cell, or select the cell and hit shift-enter.  Feel free to edit the cell, and rerun it (+ the likely . impacted cells below it.)
Best practice: Write in order as if a full program, so you can always restart and run from the top

Pandas
Most of the preprocessing code is pandas, the most popular Python data science tool (https://pandas.pydata.org ). Graphistry enterprise enables you to replace this kind of manual data wrangling code with shareable point-and-click solutions.
Graphistry intro:


Graphistry loads below in every cell that says  ...plot()


If you see a giant Graphistry logo over a gray background and nothing else, click the logo to start the Graphistry session


UI Guide: https://labs.graphistry.com/graphistry/ui.html 


Graphistry notebook examples: https://github.com/graphistry/pygraphistry


Palettes: https://labs.graphistry.com/graphistry/docs/palette.html


Try changing "... | head 100"  to  "... | head 10000"!
End of explanation
"""



#optional - add:     OR (version=* AND version != TLSv12)   

certs_a_df = splunkToPandas("""

    search index="corelight_tutorial" cert_chain_fuids{}=* 
    validation_status="certificate has expired" 
    OR validation_status="self signed certificate" 
    OR validation_status ="self signed certificate in certificate chain"
    
    
    | fields *
    | fields - _*
                                   

    | head 50000

    """,
    {'sample_ratio': 1})

print('# rows', len(certs_a_df))
certs_a_df.sample(10)

"""
Explanation: 1. Hunting through encrypted traffic

Motivation: Internal and perimeter traffic is increasingly encrypted, but we still need to look at it for reasons including auditing encryption hygiene and understanding disguised malicious traffic.
Input: SSL logs
Methodology
Search for expired, self-signed, internal CAs, old TLS versions, ...
Map out & investigate
Work through combos of version TLS 1.2 (old) and validation search
Look for funny issuers, subjects
Use JA3 to fingerprint &  whitelist good TLS; then just focus on non-JA3


Insights
1: Clear clusters of TLS version hygiene issues across the various users & applications

2: One cluster is signed... Obama?!?! 


Generalize

Build whitelist of JA3 and look for violators
For unknown certs, characterize nature of activity based on behavior like periodic beaconing, heavy back-and-forth (tunnel), heavy data movement (exfil), ...
Map structure of certs in general: services -> certs -> authorities
End of explanation
"""


hg = graphistry.hypergraph(
    certs_a_df, 
    ["id.orig_h", "id.resp_h", "uid", "ja3", "issuer", "subject"], ### "uid", "protocol", ....
    direct=True,
    opts={
        **opts,
        'EDGES': {
            'id.orig_h': ["id.resp_h", "ja3", "subject"],
            'ja3': ['id.resp_h'],
            "subject": ['id.resp_h'],
            'issuer': ['id.resp_h']
        }})

hg['graph'].bind(edge_title='category').drop_hyper_nans().color_points_by('category').color_edges_by('version').plot()

"""
Explanation: The graph:

Nodes: IPs, ja3 (TLS metadata hashes), cert subject/issuers, colored by category
Edges: Color by TLS version, title by issuer
End of explanation
"""


ntlm_a_df = splunkToPandas("""

    search index="corelight_tutorial" 
        [ search index="corelight_tutorial" ntlm | dedup uid | fields + uid  ]
    | fields * 
                                   

    | head 1000

    """,
    {'sample_ratio': 1})

print('# rows', len(ntlm_a_df))
ntlm_a_df.sample(3)

"""
Explanation: 2. Hunting Insider Threats with NTLM+SMB

Motivation:  NTLM (NT Lan Manager) logins are suspicious, especially on senstive data shares, worth auditing.
Input: NTLM + SMB + other network logs (for any other IPs/activities)
Methodology:
Seed search by NTLM activity
Get all other logs involving those UIDs

Map & audit


Insights:

Cluster 1: Sonos smart speakers seem to be opening network-shared data that have nothing to do with listening to music

Cluster 2: Second cluster -- same Workgroup domain name, yet on a rogue host


Generalize:

Map & audit NTLM and other remote logins to beginwith
From those hits, look at other file shares beyond SMB - dropbox, wiki's, ...
Map & audit file shares in general
End of explanation
"""


hg = graphistry.hypergraph(
    ntlm_a_df, 
    ["id.orig_h", "name", "id.resp_h", "path", "hostname", "domainname", "username"], ### "uid", "protocol", ....
    direct=True,
    opts={
        **opts,
        'EDGES': {
            "username": ['id.orig_h'],
            "id.orig_h": ['name', 'id.resp_h',  "hostname", "domainname"],       
            'path': ['name'],
            'hostname': ['id.resp_h'],
            'domainname': ['id.resp_h'],
            "name": ['id.resp_h'],
            "id.resp_h": ['username']
        }})
        

hg['graph'].bind(edge_title='name').drop_hyper_nans().color_points_by('category').color_edges_by('username').plot()

"""
Explanation: The graph:
Focus on representing NTLM, SMB, and generic Bro/Zeek logs.

Nodes: IPs, domains/hosts/usernames, files/paths, colored by category
Edges: Zeek events connecting them, colored by username
End of explanation
"""


dns_a_df = splunkToPandas("""

    search index="corelight_tutorial" sourcetype="conn"
    
    | eval total_bytes = orig_ip_bytes + resp_ip_bytes
    | eval log_total_bytes = log(orig_ip_bytes + resp_ip_bytes)

    | stats
    count(_time) as count,
    earliest(_time), latest(_time),
    values(answers{}) as answers,
    values(conn_state),
    values(history)
    values(issuer),
    values(ja3),
    values(last_alert),
    values(qtype_name),
    values(subject),
    max(*bytes), avg(*bytes), sum(*bytes),

    by id.orig_h, id.resp_h

    | eval duration_ms = last_time_ms - first_time_ms

    | head 50000

    """,
    {'sample_ratio': 1})

print('# rows', len(dns_a_df))
dns_a_df.sample(3)

"""
Explanation: 3. DNS Tunneling
3.A. Setup -- General DNS map
General query for looking at DNS connections with Bro/Zeek. 
Instead of showing each connection, summarize all activities across each unique 10,000 IP<>IP pairs: max bytes, first/last communication, ...
For UI work, compute the log(..) of bytes
End of explanation
"""


hg = graphistry.hypergraph(
    dns_a_df, 
    ["id.orig_h", "id.resp_h"], ### "uid", "protocol", ....
    direct=True,
    opts=opts)

hg['graph'].color_points_by('category').color_edges_by('max(log_total_bytes)', 'continuous').bind(edge_title='max(total_bytes)').plot()

"""
Explanation: Graph demo

Nodes are IPs
Edges summarize all activity per IP<>IP: first time, ... 
Color by total bytes in/out
End of explanation
"""


dns_b_df = splunkToPandas("""

    search index="corelight_tutorial" sourcetype="conn"
    
    | eval total_bytes = orig_ip_bytes + resp_ip_bytes
    | eval log_total_bytes = log(orig_ip_bytes + resp_ip_bytes)

    | eval query_length = length(query)
    | eval long_answers=mvfilter(length('answers{}') > 45)
    | eval long_answers_length = max(length(long_answers))
    | where query_length > 25 OR long_answers_length > 45


    | stats
    count(_time) as count,
    earliest(_time), latest(_time),
    values(answers{}) as answers,
    max(long_answers_length) as max_long_answers_length,
    values(conn_state),
    values(history)
    values(issuer),
    values(ja3),
    values(last_alert),
    values(subject),
    max(*bytes), avg(*bytes), sum(*bytes),
    values(qtype_name),
    first(uid),

    max(*bytes), avg(*bytes), sum(*bytes),
    
    by id.orig_h, id.resp_h, query, query_length                                

    | eval duration_ms = last_time_ms - first_time_ms
    
    | eval query=substr(query,1,100)
    | eval max_query_or_answer_length = max(query_length, max_long_answers_length)
    | sort max_query_or_answer_length desc                                           

    | head 50000

    """,
    {'sample_ratio': 1})

print('# rows', len(dns_b_df))
dns_b_df.sample(3)

"""
Explanation: 3.B. DNS Tunnel:


Motivation: DNS is a sneaky channel for hiding activity. Need to detect & unravel, whether proactive or post-breach.


Input: DNS connections


Methodology: 


Search for the top 10,000  ip->(unique dns query)->ip summaries matching tunneling heuristics:

length(query) > 25: exfil / command request
length(answer) > 45: received command 
Inspect & explain all flagged behavior
Pay attention to long and artificial looking queries & answers
Exfil: big or many queries
Command: strange responses

Tunneling: heavy back-and-forth


Insights

Two clusters of activity
One seems to be tunneling: back-and-forth

The other seems to not have answers


Generalize


The hunt continues on the identified UIDs and IPs for demo purposes, but does not reveal much. What can you find?
On a full SIEM:
  * Check periodicity (timebar) for bot vs human
  * Combine with endpoint logs to correlate proceses, file accesses, and users
  * Combine with alert logs to trace back to initial breach and subsequent behavior
End of explanation
"""


hg = graphistry.hypergraph(
    dns_b_df, 
    ["id.orig_h", "id.resp_h", "query", "answers"], ### "uid", "protocol", ....
    direct=True,
    opts={
        **opts,
        'EDGES': {
            'id.orig_h': ['query'],
            'query': ['id.resp_h'],
            'id.resp_h': ['answers'],
            'answers': ['id.orig_h']
        }})

g = hg['graph'].bind(edge_title='query').drop_hyper_nans().color_points_by('category').color_edges_by('max_query_or_answer_length', 'continuous')

g.plot()

"""
Explanation: The graph:

Nodes: IPs, queries, answers
Edges: Summaries along each  orig_h->query->resp_h->answer->orig_h

Results:
* UIDs: C3ApkJ3TwWW64DtnWb , CaAbvy2ureWe5sifRf
* IPs: 10.0.2.30 10.0.2.20  34.215.241.13 192.168.1.128
End of explanation
"""


dns_b2_df = splunkToPandas("""

    search index="corelight_tutorial" 
    C3ApkJ3TwWW64DtnWb OR CaAbvy2ureWe5sifRf OR 10.0.2.30 OR 10.0.2.20  OR 34.215.241.13 OR 192.168.1.128
    | eval time=ts
    | rename answers{} as answers
    | fields *
    | fields - _*
                                   

    | head 50000

    """,
    {'sample_ratio': 1})

print('# rows', len(dns_b2_df))
dns_b2_df.sample(3)

hg = graphistry.hypergraph(
    dns_b2_df, 
    ["id.orig_h", "id.resp_h"], ### "uid", "protocol", ....
    direct=True,
    opts=opts)

hg['graph'].bind(edge_title='sourcetype').drop_hyper_nans().color_points_by('category').color_edges_by('sourcetype').plot()

"""
Explanation: Dig into interesting UIDs and IPs 1: IP map

Surface IPs interacted with
What log types are available 
sourcetypes: conn and weird
End of explanation
"""


dns_b3_df = splunkToPandas("""

    search index="corelight_tutorial" sourcetype="conn"
    C3ApkJ3TwWW64DtnWb OR CaAbvy2ureWe5sifRf OR 10.0.2.30 OR 10.0.2.20  OR 34.215.241.13 OR 192.168.1.128
    
    | eval total_bytes = orig_ip_bytes + resp_ip_bytes
    | eval log_total_bytes = log(orig_ip_bytes + resp_ip_bytes)

    | eval query_length = length(query)
    | eval long_answers=mvfilter(length('answers{}') > 45)
    | eval long_answers_length = max(length(long_answers))
    | where query_length > 25 OR long_answers_length > 45


    | stats
    count(_time) as count,
    earliest(_time), latest(_time),
    values(answers{}) as answers,
    max(long_answers_length) as max_long_answers_length,
    values(conn_state),
    values(history)
    values(issuer),
    values(ja3),
    values(last_alert),
    values(subject),
    max(*bytes), avg(*bytes), sum(*bytes),
    values(qtype_name),
    first(uid),

    max(*bytes), avg(*bytes), sum(*bytes),
    
    by id.orig_h, id.resp_h, query, query_length                               

    | eval duration_ms = last_time_ms - first_time_ms
    
    | eval query=substr(query,1,100)
    | eval max_query_or_answer_length = max(query_length, max_long_answers_length)
    | sort max_query_or_answer_length desc                                           

    | head 50000

    """,
    {'sample_ratio': 1})

print('# rows', len(dns_b3_df))
dns_b3_df.sample(3)

hg = graphistry.hypergraph(
    dns_b3_df, 
    ["id.orig_h", "id.resp_h", "query", "answers", "first(uid)"], ### "uid", "protocol", ....
    direct=True,
    opts={
        **opts,
        'EDGES': {
            'id.orig_h': ['query'],
            'query': ['id.resp_h'],
            'id.resp_h': ['answers'],
            'answers': ['id.orig_h']
        }})

hg['graph'].bind(edge_title='query').drop_hyper_nans().color_points_by('category').color_edges_by('max_query_or_answer_length', 'continuous').plot()

"""
Explanation: Dig into interesting UIDs and IPs 2: Mostly just connections, so inspect from that perspective

Reuse DNS query from before
End of explanation
"""


mime_df = splunkToPandas("""

    search index=corelight_tutorial filename!=*.exe mime_type=application/x-dosexec                                         

    | head 200

    """,
    {'sample_ratio': 1})

print('# rows', len(dns_b3_df))
dns_b3_df.sample(3)

"""
Explanation: 4. Mimetype Mismatch


Motivation: When following an incident or doing a sweep, a common case is executable files hiding behind  extensions like ".jpeg", and brings into question the UIDs of all entities involved


Data: Multiple. Ex:


Methodology: 

Entity of interest: index=main sourcetype=corelight* filename!=*.exe mime_type=application/x-dosexec

Files that aren't named with the proper extension. Can pivot off md5/Sha1/Sha256. Can track tx_host and rx_host.


Insight:
End of explanation
"""


## Old a

"""
Explanation: The graph:
??
End of explanation
"""


