!pip install adabound
import os
import json
import random
import collections

import numpy as np
import pandas as pd
import cv2
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import signal
import adabound
def convert_image_id_2_path(image_id: str, is_train: bool = True) -> str:
    folder = "train" if is_train else "test"
    return "../input/g2net-gravitational-wave-detection/{}/{}/{}/{}/{}.npy".format(
        folder, image_id[0], image_id[1], image_id[2], image_id 
    )
train_df = pd.read_csv("../input/g2net-gravitational-wave-detection/training_labels.csv")
train_df
sns.countplot(data=train_df, x="target")
"""
Each data sample (npy file) contains 3 time series (1 for each detector) and each spans 2 sec and is sampled at 2,048 Hz.
"""
'''
# We need to suppress the high frequencies with some bandpassing:
bb, ab = butter(4, [20.*2./fs, 300.*2./fs], btype='band')
strain_H1_whitenbp = filtfilt(bb, ab, strain_H1_whiten)
strain_L1_whitenbp = filtfilt(bb, ab, strain_L1_whiten)
NR_H1_whitenbp = filtfilt(bb, ab, NR_H1_whiten)
'''

def bpf(wave, fs, fe1, fe2):
    nyq = fs / 2.0
    b, a = signal.butter(1, [fe1/nyq, fe2/npq], btype='band', fs=4096)
    return b, a
    #wave = signal.lfilter(b, a, wave)   
    #return wave
"""
# BPF
"""
# bpf可視化

def bpf_visualize(
    _id, 
    target,
    fs=4096,
    fe1=20,
    fe2=500,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    colors=("black", "red", "green")
):
    x = np.load(convert_image_id_2_path(_id))
    b, a = bpf(x, fs, fe1, fe2)
    wave = signal.lfilter(b, a, x)    
    
    #np.save( f'./output/{_id}.npy', wave)
    
    plt.figure(figsize=(16, 5))
    for i in range(3):
        X = wave[i] / wave[i].max()
        #S = np.fft.fftfreq(len(X), d=1/2048)        
        plt.subplot(1, 3, i + 1)
        plt.plot(X, color=colors[i])
        #plt.xlim(0,100)
        #plt.ylim(0,10000)
        #plt.colorbar()
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()

for i in range(3):#random.sample(train_df.index.tolist(), 3):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    bpf_visualize(_id, target)
!pip install -q nnAudio -qq
import torch
from nnAudio.Spectrogram import CQT1992v2

Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)

def visualize_sample_qtransform(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
):
    #x = np.load(('./output/' + _id + '.npy'))
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    for i in range(3):
        waves = x[i] / np.max(x[i])
        waves = torch.from_numpy(waves).float()
        image = Q_TRANSFORM(waves)
        #print(image.shape)
        
        plt.subplot(1, 3, i + 1)
        plt.imshow(image.squeeze())
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()

for i in range(3):#random.sample(train_df.index.tolist(), 5):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    #visualize_sample(_id, target)
    visualize_sample_qtransform(_id, target)
"""
# bpfデータセット作成（tfrec）
"""
!mkdir ./bpf
# waveデータを(56000, 3, 4096)に変更
import torch
from nnAudio.Spectrogram import CQT1992v2
import tensorflow as tf


Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)

def visualize_sample_qtransform(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
    fs=4096,
    fe1=20,
    fe2=500
):
    #x = np.load(('./output/' + _id + '.npy'))
    x = np.load(convert_image_id_2_path(_id))
    target = np.array([target]) * np.ones(x.shape[0]).reshape(-1,1)
    b, a = bpf(x, fs, fe1, fe2)
    wave = signal.lfilter(b, a, x) 
    #wave = np.concatenate([wave, target],axis=1) 
    return wave, target

def record2example(wave, target):
    return tf.train.Example(features=tf.train.Features(feature={
        "x": tf.train.Feature(bytes_list=tf.train.BytesList(value=[wave])),
        "y":  tf.train.Feature(bytes_list=tf.train.BytesList(value=[target]))
    }))


def write_tfrecord(images, labels, filename):
    writer = tf.compat.v1.python_io.TFRecordWriter(filename)
    for image, label in zip(images, labels):
        labels = labels.astype(np.int64)
        ex = record2example(image.tobytes(), label.tobytes())
        writer.write(ex.SerializeToString())
    


tmp = int(560000 * (1/5))

bpf_list = np.zeros((3, 3, 4096+1))

for i in range(10):#random.sample(train_df.index.tolist(), 5):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    
    wave, target = visualize_sample_qtransform(_id, target)
    write_tfrecord(wave, target, './bpf/' + _id + '.tfrec')
!ls -a
!mkdir ./output
#!rm -fr ./output
#%cd working

"""
# マッチドフィルター
"""
# メモリ爆発
# -- To calculate the PSD of the data, choose an overlap and a window (common to all detectors)
#   that minimizes "spectral leakage" https://en.wikipedia.org/wiki/Spectral_leakage
fs = 4096
NFFT = 4*fs
psd_window = np.blackman(NFFT)
# and a 50% overlap:
NOVL = NFFT/2

# define the complex template, common to both detectors:
template = (template_p + template_c*1.j) 
# We will record the time where the data match the END of the template.
etime = time+template_offset
# the length and sampling rate of the template MUST match that of the data.
datafreq = np.fft.fftfreq(template.size)*fs
df = np.abs(datafreq[1] - datafreq[0])

# to remove effects at the beginning and end of the data stretch, window the data
# https://en.wikipedia.org/wiki/Window_function#Tukey_window
try:   dwindow = signal.tukey(template.size, alpha=1./8)  # Tukey window preferred, but requires recent scipy version 
except: dwindow = signal.blackman(template.size)          # Blackman window OK if Tukey is not available

# prepare the template fft.
template_fft = np.fft.fft(template*dwindow) / fs

# loop over the detectors
dets = ['H1', 'L1']
for det in dets:

    if det is 'L1': data = strain_L1.copy()
    else:           data = strain_H1.copy()

    # -- Calculate the PSD of the data.  Also use an overlap, and window:
    data_psd, freqs = mlab.psd(data, Fs = fs, NFFT = NFFT, window=psd_window, noverlap=NOVL)

    # Take the Fourier Transform (FFT) of the data and the template (with dwindow)
    data_fft = np.fft.fft(data*dwindow) / fs

    # -- Interpolate to get the PSD values at the needed frequencies
    power_vec = np.interp(np.abs(datafreq), freqs, data_psd)

    # -- Calculate the matched filter output in the time domain:
    # Multiply the Fourier Space template and data, and divide by the noise power in each frequency bin.
    # Taking the Inverse Fourier Transform (IFFT) of the filter output puts it back in the time domain,
    # so the result will be plotted as a function of time off-set between the template and the data:
    optimal = data_fft * template_fft.conjugate() / power_vec
    optimal_time = 2*np.fft.ifft(optimal)*fs

    # -- Normalize the matched filter output:
    # Normalize the matched filter output so that we expect a value of 1 at times of just noise.
    # Then, the peak of the matched filter output will tell us the signal-to-noise ratio (SNR) of the signal.
    sigmasq = 1*(template_fft * template_fft.conjugate() / power_vec).sum() * df
    sigma = np.sqrt(np.abs(sigmasq))
    SNR_complex = optimal_time/sigma

    # shift the SNR vector by the template length so that the peak is at the END of the template
    peaksample = int(data.size / 2)  # location of peak in the template
    SNR_complex = np.roll(SNR_complex,peaksample)
    SNR = abs(SNR_complex)

    # find the time and SNR value at maximum:
    indmax = np.argmax(SNR)
    timemax = time[indmax]
    SNRmax = SNR[indmax]

    # Calculate the "effective distance" (see FINDCHIRP paper for definition)
    # d_eff = (8. / SNRmax)*D_thresh
    d_eff = sigma / SNRmax
    # -- Calculate optimal horizon distnace
    horizon = sigma/8

    # Extract time offset and phase at peak
    phase = np.angle(SNR_complex[indmax])
    offset = (indmax-peaksample)

    # apply time offset, phase, and d_eff to template 
    template_phaseshifted = np.real(template*np.exp(1j*phase))    # phase shift the template
    template_rolled = np.roll(template_phaseshifted,offset) / d_eff  # Apply time offset and scale amplitude
    
    # Whiten and band-pass the template for plotting
    template_whitened = whiten(template_rolled,interp1d(freqs, data_psd),dt)  # whiten the template
    template_match = filtfilt(bb, ab, template_whitened) / normalization # Band-pass the template
    
    print('For detector {0}, maximum at {1:.4f} with SNR = {2:.1f}, D_eff = {3:.2f}, horizon = {4:0.1f} Mpc' 
          .format(det,timemax,SNRmax,d_eff,horizon))

    if make_plots:

        # plotting changes for the detectors:
        if det is 'L1': 
            pcolor='g'
            strain_whitenbp = strain_L1_whitenbp
            template_L1 = template_match.copy()
        else:
            pcolor='r'
            strain_whitenbp = strain_H1_whitenbp
            template_H1 = template_match.copy()

        # -- Plot the result
        plt.figure(figsize=(10,8))
        plt.subplot(2,1,1)
        plt.plot(time-timemax, SNR, pcolor,label=det+' SNR(t)')
        #plt.ylim([0,25.])
        plt.grid('on')
        plt.ylabel('SNR')
        plt.xlabel('Time since {0:.4f}'.format(timemax))
        plt.legend(loc='upper left')
        plt.title(det+' matched filter SNR around event')

        # zoom in
        plt.subplot(2,1,2)
        plt.plot(time-timemax, SNR, pcolor,label=det+' SNR(t)')
        plt.grid('on')
        plt.ylabel('SNR')
        plt.xlim([-0.15,0.05])
        #plt.xlim([-0.3,+0.3])
        plt.grid('on')
        plt.xlabel('Time since {0:.4f}'.format(timemax))
        plt.legend(loc='upper left')
        plt.savefig(eventname+"_"+det+"_SNR."+plottype)

        plt.figure(figsize=(10,8))
        plt.subplot(2,1,1)
        plt.plot(time-tevent,strain_whitenbp,pcolor,label=det+' whitened h(t)')
        plt.plot(time-tevent,template_match,'k',label='Template(t)')
        plt.ylim([-10,10])
        plt.xlim([-0.15,0.05])
        plt.grid('on')
        plt.xlabel('Time since {0:.4f}'.format(timemax))
        plt.ylabel('whitened strain (units of noise stdev)')
        plt.legend(loc='upper left')
        plt.title(det+' whitened data around event')

        plt.subplot(2,1,2)
        plt.plot(time-tevent,strain_whitenbp-template_match,pcolor,label=det+' resid')
        plt.ylim([-10,10])
        plt.xlim([-0.15,0.05])
        plt.grid('on')
        plt.xlabel('Time since {0:.4f}'.format(timemax))
        plt.ylabel('whitened strain (units of noise stdev)')
        plt.legend(loc='upper left')
        plt.title(det+' Residual whitened data after subtracting template around event')
        plt.savefig(eventname+"_"+det+"_matchtime."+plottype)
                 
        # -- Display PSD and template
        # must multiply by sqrt(f) to plot template fft on top of ASD:
        plt.figure(figsize=(10,6))
        template_f = np.absolute(template_fft)*np.sqrt(np.abs(datafreq)) / d_eff
        plt.loglog(datafreq, template_f, 'k', label='template(f)*sqrt(f)')
        plt.loglog(freqs, np.sqrt(data_psd),pcolor, label=det+' ASD')
        plt.xlim(20, fs/2)
        plt.ylim(1e-24, 1e-20)
        plt.grid()
        plt.xlabel('frequency (Hz)')
        plt.ylabel('strain noise ASD (strain/rtHz), template h(f)*rt(f)')
        plt.legend(loc='upper left')
        plt.title(det+' ASD and template around event')
        plt.savefig(eventname+"_"+det+"_matchfreq."+plottype)

"""
# FFT可視化
"""
# FFT可視化
def FFT_visualize(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    colors=("black", "red", "green"),
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    for i in range(3):
        X = np.fft.fft(x[i] / x[i].max())
        S = np.fft.fftfreq(len(X), d=1/2048)
        
        #Xdb = librosa.amplitude_to_db(abs(X))
        plt.subplot(1, 3, i + 1)
        #librosa.display.specshow(S, sr=2048, x_axis="time", y_axis="hz", vmin=-30, vmax=50) 
        plt.plot(S, np.abs(X), color=colors[i])
        plt.xlim(0,100)
        #plt.colorbar()
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()

for i in range(3):#random.sample(train_df.index.tolist(), 10):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    FFT_visualize(_id, target)
"""
# サイン波合算スペクトル
"""
# サイン波の合算を可視化
def visualize_sample(
    _id, 
    target, 
    colors=("black", "red", "green"), 
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo")
):
    path = convert_image_id_2_path(_id)
    x = np.load(path)
    sum_x = x.sum(axis=0)
    sum_x = (sum_x/sum_x.max())
    print(sum_x.shape)
    plt.figure(figsize=(16, 7))
    plt.plot(sum_x)
    plt.show()
    X = np.fft.fft(sum_x)
    S = np.fft.fftfreq(len(X), d=1/2048)   
    plt.plot(S, np.abs(X))
    plt.xlim(0,100)

for i in range(1):#random.sample(train_df.index.tolist(), 3):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    visualize_sample(_id, target)
# サイン波の合算をスペクトル可視化
import librosa 
import librosa.display

def visualize_sample(
    _id, 
    target, 
    colors=("black", "red", "green"), 
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo")
):
    path = convert_image_id_2_path(_id)
    x = np.load(path)
    sum_x = x.sum(axis=0)
    sum_x = (sum_x/sum_x.max())
    print(sum_x.shape)
    
    X = librosa.stft(sum_x / sum_x.max())
    Xdb = librosa.amplitude_to_db(abs(X))
    
    #plt.figure(figsize=(16, 7))
    librosa.display.specshow(Xdb, sr=2048, x_axis="time", y_axis="hz", vmin=-30, vmax=50) 
    plt.show()

for i in random.sample(train_df.index.tolist(), 1):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    visualize_sample(_id, target)
"""
# こっから普通①
"""
def visualize_sample(
    _id, 
    target, 
    colors=("black", "red", "green"), 
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo")
):
    path = convert_image_id_2_path(_id)
    x = np.load(path)
    plt.figure(figsize=(16, 7))
    for i in range(3):
        plt.subplot(4, 1, i + 1)
        plt.plot(x[i], color=colors[i])
        plt.legend([signal_names[i]], fontsize=12, loc="lower right")
        
        plt.subplot(4, 1, 4)
        plt.plot(x[i], color=colors[i])
    
    plt.subplot(4, 1, 4)
    plt.legend(signal_names, fontsize=12, loc="lower right")

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()
for i in range(3):#random.sample(train_df.index.tolist(), 3):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    visualize_sample(_id, target)
import librosa
import librosa.display
def visualize_sample_spectogram(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo")
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    sum_x = np.zeros((x.shape[0], x.shape[1]))
    for i in range(3):
        X = librosa.stft(x[i] / x[i].max())
        Xdb = librosa.amplitude_to_db(abs(X))
        plt.subplot(1, 3, i + 1)
        librosa.display.specshow(Xdb, sr=2048, x_axis="time", y_axis="hz", vmin=-30, vmax=50) 
        plt.colorbar()
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()
for i in range(3):#random.sample(train_df.index.tolist(), 3):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    visualize_sample_spectogram(_id, target)
def visualize_sample_mfcc(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    for i in range(3):
        mfccs = librosa.feature.mfcc(x[i] / x[i].max(), sr=sr)
        plt.subplot(1, 3, i + 1)
        librosa.display.specshow(mfccs, sr=sr, x_axis="time", vmin=-200, vmax=50, cmap="coolwarm")
        plt.title(signal_names[i], fontsize=14)
        plt.colorbar()

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()
for i in random.sample(train_df.index.tolist(), 3):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    visualize_sample_mfcc(_id, target)
# !pip install pycbc -qq
# import pycbc.types
!pip install -q nnAudio -qq
import torch
from nnAudio.Spectrogram import CQT1992v2
Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)

def visualize_sample_qtransform(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    for i in range(3):
        waves = x[i] / np.max(x[i])
        waves = torch.from_numpy(waves).float()
        image = Q_TRANSFORM(waves)
        #print(image.shape)
        
        plt.subplot(1, 3, i + 1)
        plt.imshow(image.squeeze())
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()

for i in range(10):#random.sample(train_df.index.tolist(), 5):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    #visualize_sample(_id, target)
    visualize_sample_qtransform(_id, target)
"""
# CQTを平均化フィルターで実装
"""
import cv2
import numpy as np

# kernel of blur filter
# カーネル（縦方向の輪郭検出用）
kernel = np.array([[1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9]])

Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)

def visualize_sample_qtransform(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    waves = np.zeros((x.shape[0], x.shape[1]))
    for i in range(3):
        waves = x[i] / np.max(x[i])
        waves = torch.from_numpy(waves).float()
        waves = Q_TRANSFORM(waves)
        #waves = waves.numpy()
        #waves = cv2.filter2D(waves, -1, kernel)
        #waves = torch.from_numpy(waves).float()
        return waves
        
        plt.subplot(1, 3, i + 1)
        plt.imshow(waves.squeeze())
        plt.title(signal_names[i], fontsize=14)

    plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    plt.show()

#avg_filter = np.zeros((len(train_df), 3, 4096))

for i in range(10):#random.sample(train_df.index.tolist(), 5):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    #visualize_sample(_id, target)
    avg_filter = visualize_sample_qtransform(_id, target)
    print(avg_filter.shape)
"""
# 平均化フィルターを更に鮮鋭化フィルターをかける
"""
import cv2
import numpy as np

# kernel of blur filter
# カーネル（縦方向の輪郭検出用）
kernel = np.array([[1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9]])

Q_TRANSFORM = CQT1992v2(sr=2048, fmin=20, fmax=1024, hop_length=32)

# 鮮鋭化
def make_sharp_kernel(k: int):
    return np.array([
    [-k / 9, -k / 9, -k / 9],
    [-k / 9, 1 + 8 * k / 9, k / 9],
    [-k / 9, -k / 9, -k / 9]], np.float32)

def visualize_sample_qtransform(
    _id, 
    target,
    signal_names=("LIGO Hanford", "LIGO Livingston", "Virgo"),
    sr=2048,
    kernel_size=5
):
    x = np.load(convert_image_id_2_path(_id))
    plt.figure(figsize=(16, 5))
    waves = np.zeros((x.shape[0], x.shape[1]))
    for i in range(3):
        waves = x[i] / np.max(x[i])
        waves = torch.from_numpy(waves).float()
        image = Q_TRANSFORM(waves)
        image = image.numpy()
        # 平均化フィルター
        image = cv2.filter2D(image, -1, kernel)
        
        # 鮮鋭化
        #kernel_filter = make_sharp_kernel(1)
        #image = cv2.filter2D(image, -1, kernel_filter)
        
        # メディアンフィルター
        #image = cv2.medianBlur(image, kernel_size)
        #image = torch.from_numpy(image).float()
        
        #plt.subplot(1, 3, i + 1)
        #plt.imshow(image.squeeze())
        #plt.title(signal_names[i], fontsize=14)

    #plt.suptitle(f"id: {_id} target: {target}", fontsize=16)
    #plt.show()
    

for i in range(10):#random.sample(train_df.index.tolist(), 5):
    _id = train_df.iloc[i]["id"]
    target = train_df.iloc[i]["target"]

    #visualize_sample(_id, target)
    visualize_sample_qtransform(_id, target)

# 平均化が違いを１番はっきり示している
# 鮮鋭化は輪郭がはっきりするらしがボケている
# メディアンフィルターをかけるとよりボケて見える
"""

Submissions are evaluated on [area under the ROC curve](https://en.wikipedia.org/wiki/Receiver_operating_characteristic) between the predicted probability and the observed target.

"""
"""
# こっから普通②
"""
from sklearn.metrics import roc_auc_score, roc_curve, auc

list_y_true = [
    [1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.],
    [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.], #  IMBALANCE
]
list_y_pred = [
    [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
    [0.9, 0.9, 0.9, 0.9, 0.1, 0.9, 0.9, 0.1, 0.9, 0.1, 0.1, 0.5],
    [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], #  IMBALANCE
]

for y_true, y_pred in zip(list_y_true, list_y_pred):
    fpr, tpr, _ = roc_curve(y_true, y_pred)
    roc_auc = auc(fpr, tpr)

    plt.figure(figsize=(5, 5))
    plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.2f)' % roc_auc)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([-0.01, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic example')
    plt.legend(loc="lower right")
    plt.show()

submission = pd.read_csv("../input/g2net-gravitational-wave-detection/sample_submission.csv")
submission.to_csv("submission.csv", index=False)
del submission
!pip install efficientnet_pytorch -qq
import time

import torch
from torch import nn
from torch.utils import data as torch_data
from sklearn import model_selection as sk_model_selection
from torch.nn import functional as torch_functional
from torch.autograd import Variable
import efficientnet_pytorch
from sklearn.model_selection import StratifiedKFold
def set_seed(seed):
    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(seed)
        torch.backends.cudnn.deterministic = True


set_seed(42)
class DataRetriever(torch_data.Dataset):
    def __init__(self, paths, targets):
        self.paths = paths
        self.targets = targets
        self.fs=4096
        self.fe1=20 
        self.fe2=500
        self.kernel = np.array([[1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9],
                   [1/9, 1/9, 1/9]])

        
        self.q_transform = CQT1992v2(
            sr=2048, fmin=20, fmax=1024, hop_length=32
        )
          
    def __len__(self):
        return len(self.paths)
    
    def bpf(self, wave):
        nyq = self.fs / 2.0
        b, a = signal.butter(1, [self.fe1, self.fe2], btype='band', fs=self.fs)
        wave = signal.lfilter(b, a, wave)   
        return wave
    
    def get_qtransform(self, x):
        image = []
        for i in range(3):
            # bpf
            waves = self.bpf(x[i])
            waves = waves / np.max(waves)
            waves = torch.from_numpy(waves).float()
            channel = self.q_transform(waves).squeeze().numpy()
            # 平均化フィルター
            #channel = cv2.filter2D(channel, -1, self.kernel).squeeze()
            '''
            waves = torch.from_numpy(waves).float()
            waves = Q_TRANSFORM(waves)
            waves = waves.numpy()
            waves = cv2.filter2D(waves, -1, kernel)
            waves = torch.from_numpy(waves).float()
            '''
            image.append(channel)
            
        return torch.tensor(image).float()
    
    def __getitem__(self, index):
        file_path = convert_image_id_2_path(self.paths[index])
        x = np.load(file_path)
        image = self.get_qtransform(x)
        
        y = torch.tensor(self.targets[index], dtype=torch.float)
            
        return {"X": image, "y": y}
class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = efficientnet_pytorch.EfficientNet.from_pretrained("efficientnet-b7")
        n_features = self.net._fc.in_features
        self.net._fc = nn.Linear(in_features=n_features, out_features=1, bias=True)
    
    def forward(self, x):
        out = self.net(x)
        return out
class LossMeter:
    def __init__(self):
        self.avg = 0
        self.n = 0

    def update(self, val):
        self.n += 1
        # incremental update
        self.avg = val / self.n + (self.n - 1) / self.n * self.avg

        
class AccMeter:
    def __init__(self):
        self.avg = 0
        self.n = 0
        
    def update(self, y_true, y_pred):
        y_true = y_true.cpu().numpy().astype(int)
        y_pred = y_pred.cpu().numpy() >= 0
        last_n = self.n
        self.n += len(y_true)
        true_count = np.sum(y_true == y_pred)
        # incremental update
        self.avg = true_count / self.n + last_n / self.n * self.avg
class Trainer:
    def __init__(
        self, 
        model, 
        device, 
        optimizer, 
        criterion, 
        loss_meter, 
        score_meter
    ):
        self.model = model 
        self.device = device 
        self.optimizer = optimizer 
        self.criterion = criterion 
        self.loss_meter = loss_meter 
        self.score_meter = score_meter 
        
        self.best_valid_score = -np.inf
        self.n_patience = 0
        
        self.messages = {
            "epoch": "[Epoch {}: {}] loss: {:.5f}, score: {:.5f}, time: {} s",
            "checkpoint": "The score improved from {:.5f} to {:.5f}. Save model to '{}'",
            "patience": "\nValid score didn't improve last {} epochs."
        }
    
    # 可視化に近い？
    '''
    #=========================================3
    '''
    def fit(self, epochs, train_loader, valid_loader, save_path, patience):        
        for n_epoch in range(1, epochs + 1):
            self.n_epoch = n_epoch
            self.info_message("EPOCH: {}", n_epoch)
            
            train_loss, train_score, train_time = self.train_epoch(train_loader)
            valid_loss, valid_score, valid_time = self.valid_epoch(valid_loader)
            
            self.info_message(
                self.messages["epoch"], "Train", n_epoch, train_loss, train_score, train_time
            )
            
            self.info_message(
                self.messages["epoch"], "Valid", n_epoch, valid_loss, valid_score, valid_time
            )

            if True:
#             if self.best_valid_score < valid_score:
                self.info_message(
                    self.messages["checkpoint"], self.best_valid_score, valid_score, save_path + str(self.n_epoch - 1) +'.pth')
                self.best_valid_score = valid_score
                self.save_model(n_epoch, save_path)
                self.n_patience = 0
            else:
                self.n_patience += 1
            
            if self.n_patience >= patience:
                self.info_message(self.messages["patience"], patience)
                break
            
    # 学習はこっち
    def train_epoch(self, train_loader):
        self.model.train()
        t = time.time()
        train_loss = self.loss_meter()
        train_score = self.score_meter()
        self.train_loss_list = []
        self.train_score_list = []
        
        for step, batch in enumerate(train_loader, start=1):
            X = batch["X"].to(self.device)
            targets = batch["y"].to(self.device)
            # 勾配の初期化
            self.optimizer.zero_grad()
            # 予測データを抽出している
            outputs = self.model(X).squeeze(1) # 次元1を潰す
            
            loss = self.criterion(outputs, targets)
            # 誤差逆伝播開始
            loss.backward()
            
            '''
            #=========================================④
            ロスを開始してるのにloss?detexhで区切っているものを出している？
            '''
            train_loss.update(loss.detach().item())
            train_score.update(targets, outputs.detach())
            
            # 勾配の更新
            self.optimizer.step()
            
            _loss, _score = train_loss.avg, train_score.avg
            message = 'Train Step {}/{}, train_loss: {:.5f}, train_score: {:.5f}'
            self.info_message(message, step, len(train_loader), _loss, _score, end="\r")
        
        self.train_loss_list.append(train_loss.avg)
        self.train_score_list.append(train_score.avg)
        
        return train_loss.avg, train_score.avg, int(time.time() - t)
    
    def valid_epoch(self, valid_loader):
        self.model.eval()
        t = time.time()
        valid_loss = self.loss_meter()
        valid_score = self.score_meter()
        self.valid_loss_list = []
        self.valid_score_list = []

        for step, batch in enumerate(valid_loader, 1):
            with torch.no_grad():
                X = batch["X"].to(self.device)
                targets = batch["y"].to(self.device)

                outputs = self.model(X).squeeze(1)
                loss = self.criterion(outputs, targets)

                valid_loss.update(loss.detach().item())
                valid_score.update(targets, outputs)
                
            _loss, _score = valid_loss.avg, valid_score.avg
            message = 'Valid Step {}/{}, valid_loss: {:.5f}, valid_score: {:.5f}'
            self.info_message(message, step, len(valid_loader), _loss, _score, end="\r")
        
        return valid_loss.avg, valid_score.avg, int(time.time() - t)
    
    def save_model(self, n_epoch, save_path, extension='.pth'):
        torch.save(
            {
                "model_state_dict": self.model.state_dict(),
                "optimizer_state_dict": self.optimizer.state_dict(),
                "best_valid_score": self.best_valid_score,
                "n_epoch": n_epoch, 
            },
            save_path + str(self.n_epoch - 1) + extension
        )
    
    @staticmethod
    def info_message(message, *args, end="\n"):
        print(message.format(*args), end=end)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
'''
class get_lr_callback:
    def __init__(self, epoch,  BATCH_SIZE = 256):
        self.lr_start = 0.0001
        self.lr_max = 0.000015 * BATCH_SIZE
        self.lr_min = 0.0000001
        self.lr_ramp_ep = 3
        self.lr_sus_ep = 0
        self.lr_decay = 0.7
        self.epoch = epoch
   
    def lrfn(self):
        if self.epoch < self.lr_ramp_ep:
            lr = (self.lr_max - self.lr_start) / self.lr_ramp_ep * self.epoch + self.lr_start   
        elif self.epoch < self.lr_ramp_ep + self.lr_sus_ep:
            lr = self.lr_max    
        else:
            lr = (self.lr_max - self.lr_min) * self.lr_decay**(self.epoch - self.lr_ramp_ep - self.lr_sus_ep) + self.lr_min    
        return lr
'''
import psutil 

# メモリ容量を取得
mem = psutil.virtual_memory() 
print(mem.total)

# メモリ使用量を取得 
print(mem.used)

# メモリ空き容量を取得 
print(mem.available)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")


skf = StratifiedKFold(n_splits=2, random_state=42, shuffle=True)
for fold, (train_index, valid_index) in enumerate(skf.split(train_df, train_df["target"])):
    train_X = train_df.iloc[train_index]
    valid_X = train_df.iloc[valid_index] # Reduce calculation time
    #print(train_X.shape, valid_X.shape)

    train_data_retriever = DataRetriever(
        train_X["id"].values, 
        train_X["target"].values, 
    )

    valid_data_retriever = DataRetriever(
        valid_X["id"].values, 
        valid_X["target"].values,
    )

    train_loader = torch_data.DataLoader(
        train_data_retriever,
        batch_size=32,
        shuffle=True,
        num_workers=2,
    )

    valid_loader = torch_data.DataLoader(
        valid_data_retriever, 
        batch_size=32,
        shuffle=False,
        num_workers=2,
    )

    model = Model()
    model.to(device)

    #optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
    optimizer = adabound.AdaBound(model.parameters(), lr=0.001, final_lr=0.5)
    criterion = torch_functional.binary_cross_entropy_with_logits

    trainer = Trainer(
        model, 
        device,
        optimizer, 
        criterion, 
        LossMeter, 
        AccMeter
    )

    history = trainer.fit(
        1, 
        train_loader, 
        valid_loader, 
        'bpf_only-pth',
        #f"best-model-{fold}.pth",
        100
    )
   
models = []
for i in range(2):
    model = Model()
    model.to(device)
    
    checkpoint = torch.load(f"best-model-{i}.pth")
    model.load_state_dict(checkpoint["model_state_dict"])
    model.eval()
    
    models.append(model)
class DataRetriever(torch_data.Dataset):
    def __init__(self, paths):
        self.paths = paths

        self.q_transform = CQT1992v2(
            sr=2048, fmin=20, fmax=1024, hop_length=32
        )
          
    def __len__(self):
        return len(self.paths)
    
    def __get_qtransform(self, x):
        image = []
        for i in range(3):
            waves = x[i] / np.max(x[i])
            waves = torch.from_numpy(waves).float()
            channel = self.q_transform(waves).squeeze().numpy()
            image.append(channel)
            
        return torch.tensor(image).float()
    
    def __getitem__(self, index):
        file_path = convert_image_id_2_path(self.paths[index], is_train=False)
        x = np.load(file_path)
        image = self.__get_qtransform(x)
            
        return {"X": image, "id": self.paths[index]}
test_data_retriever = DataRetriever(
    submission["id"].values, 
)

test_loader = torch_data.DataLoader(
    test_data_retriever,
    batch_size=32,
    shuffle=False,
    num_workers=8,
)
y_pred = []
ids = []

for e, batch in enumerate(test_loader):
    print(f"{e}/{len(test_loader)}", end="\r")
    with torch.no_grad():
        tmp_pred = np.zeros((batch["X"].shape[0], ))
        for model in models:
            tmp_res = torch.sigmoid(model(batch["X"].to(device))).cpu().numpy().squeeze()
            tmp_pred += tmp_res / 2
        y_pred.extend(tmp_pred)
        ids.extend(batch["id"])
submission = pd.DataFrame({"id": ids, "target": y_pred})
submission.to_csv("model_submission.csv", index=False)
submission
"""
### WORK IN PROGRESS ...
"""