[FRAUD] 데이터정리 시도(8.14-망함 tr/test_mask 만들어봄)

imports

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt 
import networkx as nx
import sklearn
import torch

# sklearn
from sklearn import model_selection # split함수이용
from sklearn import ensemble # RF,GBM
from sklearn import metrics 

# embedding 
from node2vec import Node2Vec
from node2vec.edges import HadamardEmbedder, AverageEmbedder, WeightedL1Embedder, WeightedL2Embedder

def build_graph_bipartite(df_input, graph_type=nx.Graph()):
    df=df_input.copy()
    mapping={x:node_id for node_id, x in enumerate(set(df["cc_num"].values.tolist()+\
                                                      df["merchant"].values.tolist()))}
    
    df["from"]=df["cc_num"].apply(lambda x:mapping[x])  #엣지의 출발점
    df["to"]=df["merchant"].apply(lambda x:mapping[x])  #엣지의 도착점
    
    df = df[['from', 'to', "amt", "is_fraud"]].groupby(['from','to']).agg({"is_fraud":"sum","amt":"sum"}).reset_index()
    df["is_fraud"]=df["is_fraud"].apply(lambda x:1 if x>0 else 0)
    
    G=nx.from_edgelist(df[["from","to"]].values, create_using=graph_type)
    
    nx.set_edge_attributes(G,{(int(x["from"]),int(x["to"])):x["is_fraud"] for idx, x in df[["from","to","is_fraud"]].iterrows()}, "label")  #엣지 속성 설정,각 속성의 사기 여부부     
    nx.set_edge_attributes(G,{(int(x["from"]),int(x["to"])):x["amt"] for idx,x in df[["from","to","amt"]].iterrows()}, "weight") # 엣지 속성 설정, 각 엣지의 거래 금액

    return G


def build_graph_tripartite(df_input, graph_type=nx.Graph()):
    df=df_input.copy()
    mapping={x:node_id for node_id, x in enumerate(set(df.index.values.tolist() + 
                                                       df["cc_num"].values.tolist() +
                                                       df["merchant"].values.tolist()))}
    df["in_node"]= df["cc_num"].apply(lambda x: mapping[x])
    df["out_node"]=df["merchant"].apply(lambda x:mapping[x])
    
        
    G=nx.from_edgelist([(x["in_node"], mapping[idx]) for idx, x in df.iterrows()] +\
                        [(x["out_node"], mapping[idx]) for idx, x in df.iterrows()], create_using=graph_type)
    
    nx.set_edge_attributes(G,{(x["in_node"], mapping[idx]):x["is_fraud"] for idx, x in df.iterrows()}, "label")     
    nx.set_edge_attributes(G,{(x["out_node"], mapping[idx]):x["is_fraud"] for idx, x in df.iterrows()}, "label")   
    nx.set_edge_attributes(G,{(x["in_node"], mapping[idx]):x["amt"] for idx, x in df.iterrows()}, "weight")  
    nx.set_edge_attributes(G,{(x["out_node"], mapping[idx]):x["amt"] for idx, x in df.iterrows()}, "weight")

    return G
    
    
def down_sample_textbook(df):
    df_majority = df[df.is_fraud==0].copy()
    df_minority = df[df.is_fraud==1].copy()
    df_maj_dowsampled = sklearn.utils.resample(df_majority, n_samples=len(df_minority), replace=False, random_state=42)
    df_downsampled = pd.concat([df_minority, df_maj_dowsampled])
    return df_downsampled

def embedding(Graph):
    # Graph -> X (feature)
    _edgs = list(Graph.edges)
    subGraph = Graph.edge_subgraph([_edgs[x] for x in range(len(Graph.edges))]).copy()
    subGraph.add_nodes_from(list(set(Graph.nodes) - set(subGraph.nodes)))    
    embedded = AverageEmbedder(Node2Vec(subGraph, weight_key='weight').fit(window=10).wv)
    X = [embedded[str(_edgs[x][0]), str(_edgs[x][1])] for x in range(len(Graph.edges))]
    # Graph -> y (label)
    y = np.array(list(nx.get_edge_attributes(Graph, "label").values()))
    return X,y 

def anal(df):
    Graph = build_graph_bipartite(df)
    X,XX,y,yy = embedding(Graph)
    lrnr = RandomForestClassifier(n_estimators=100, random_state=42) 
    lrnr.fit(X,y)
    yyhat = lrnr.predict(XX)
    df = pd.DataFrame({
        'acc':[sklearn.metrics.accuracy_score(yy,yyhat)], 
        'pre':[sklearn.metrics.precision_score(yy,yyhat)], 
        'rec':[sklearn.metrics.recall_score(yy,yyhat)],
        'f1':[sklearn.metrics.f1_score(yy,yyhat)]}
    )    
    return df

def our_sampling1(df):
    cus_list = set(df.query('is_fraud==1').cc_num.tolist())
    return df.query("cc_num in @ cus_list")

fraudTrain = pd.read_csv("~/Desktop/fraudTrain.csv").iloc[:,1:]

fraudTrain = fraudTrain.assign(trans_date_trans_time= list(map(lambda x: pd.to_datetime(x), fraudTrain.trans_date_trans_time)))
fraudTrain

	trans_date_trans_time	cc_num	merchant	category	amt	first	last	gender	street	city	...	lat	long	city_pop	job	dob	trans_num	unix_time	merch_lat	merch_long	is_fraud
0	2019-01-01 00:00:00	2.703190e+15	fraud_Rippin, Kub and Mann	misc_net	4.97	Jennifer	Banks	F	561 Perry Cove	Moravian Falls	...	36.0788	-81.1781	3495	Psychologist, counselling	1988-03-09	0b242abb623afc578575680df30655b9	1325376018	36.011293	-82.048315	0
1	2019-01-01 00:00:00	6.304230e+11	fraud_Heller, Gutmann and Zieme	grocery_pos	107.23	Stephanie	Gill	F	43039 Riley Greens Suite 393	Orient	...	48.8878	-118.2105	149	Special educational needs teacher	1978-06-21	1f76529f8574734946361c461b024d99	1325376044	49.159047	-118.186462	0
2	2019-01-01 00:00:00	3.885950e+13	fraud_Lind-Buckridge	entertainment	220.11	Edward	Sanchez	M	594 White Dale Suite 530	Malad City	...	42.1808	-112.2620	4154	Nature conservation officer	1962-01-19	a1a22d70485983eac12b5b88dad1cf95	1325376051	43.150704	-112.154481	0
3	2019-01-01 00:01:00	3.534090e+15	fraud_Kutch, Hermiston and Farrell	gas_transport	45.00	Jeremy	White	M	9443 Cynthia Court Apt. 038	Boulder	...	46.2306	-112.1138	1939	Patent attorney	1967-01-12	6b849c168bdad6f867558c3793159a81	1325376076	47.034331	-112.561071	0
4	2019-01-01 00:03:00	3.755340e+14	fraud_Keeling-Crist	misc_pos	41.96	Tyler	Garcia	M	408 Bradley Rest	Doe Hill	...	38.4207	-79.4629	99	Dance movement psychotherapist	1986-03-28	a41d7549acf90789359a9aa5346dcb46	1325376186	38.674999	-78.632459	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1048570	2020-03-10 16:07:00	6.011980e+15	fraud_Fadel Inc	health_fitness	77.00	Haley	Wagner	F	05561 Farrell Crescent	Annapolis	...	39.0305	-76.5515	92106	Accountant, chartered certified	1943-05-28	45ecd198c65e81e597db22e8d2ef7361	1362931649	38.779464	-76.317042	0
1048571	2020-03-10 16:07:00	4.839040e+15	fraud_Cremin, Hamill and Reichel	misc_pos	116.94	Meredith	Campbell	F	043 Hanson Turnpike	Hedrick	...	41.1826	-92.3097	1583	Geochemist	1999-06-28	c00ce51c6ebb7657474a77b9e0b51f34	1362931670	41.400318	-92.726724	0
1048572	2020-03-10 16:08:00	5.718440e+11	fraud_O'Connell, Botsford and Hand	home	21.27	Susan	Mills	F	005 Cody Estates	Louisville	...	38.2507	-85.7476	736284	Engineering geologist	1952-04-02	17c9dc8b2a6449ca2473726346e58e6c	1362931711	37.293339	-84.798122	0
1048573	2020-03-10 16:08:00	4.646850e+18	fraud_Thompson-Gleason	health_fitness	9.52	Julia	Bell	F	576 House Crossroad	West Sayville	...	40.7320	-73.1000	4056	Film/video editor	1990-06-25	5ca650881b48a6a38754f841c23b77ab	1362931718	39.773077	-72.213209	0
1048574	2020-03-10 16:08:00	2.283740e+15	fraud_Buckridge PLC	misc_pos	6.81	Shannon	Williams	F	9345 Spencer Junctions Suite 183	Alpharetta	...	34.0770	-84.3033	165556	Prison officer	1997-12-27	8d0a575fe635bbde12f1a2bffc126731	1362931730	33.601468	-83.891921	0

1048575 rows × 22 columns

시도

_df1 = fraudTrain[fraudTrain["is_fraud"] == 0].sample(frac=0.20, random_state=42)
_df2 = fraudTrain[fraudTrain["is_fraud"] == 1]
df02 = pd.concat([_df1,_df2])
df02.shape

(214520, 22)

df50 = down_sample_textbook(df02)
df50.shape

(12012, 22)

12012*12012

144288144

고려할 것(230810)

• df50 의 shape이 12000개 이므로 9000개의 T, 3000개의 F를 train mask로 만들자.

• 고객정보가 동일하면 edge를 1로, 아니면 0으로 놓고 1에대한 weight를 만들자.

• g(V,E,W)에서의 weight

df50 = df50.reset_index()

N = len(df50)

이건 weight?

edge_index_list = []
for i in range(N):
    for j in range(N):
        time_difference = (df50['trans_date_trans_time'][i] - df50['trans_date_trans_time'][j]).total_seconds()
        edge_index_list.append([i, j, time_difference])

edge_index_list[:5]

[[0, 0, 0.0],
 [0, 1, -2460.0],
 [0, 2, -7140.0],
 [0, 3, -9120.0],
 [0, 4, -10140.0]]

np.save('edge_index_list_50.npy', edge_index_list)

loaded_data = np.load('edge_index_list_50.npy')

edge_index = np.array(edge_index_list)
edge_index[:,2] = np.abs(edge_index[:,2])
theta = edge_index[:,2].mean()
theta

12238996.895508753

edge_index[:,2] = (np.exp(-edge_index[:,2]/theta)!=1) * np.exp(-edge_index[:,2]/theta)
edge_index

array([[0.00000000e+00, 0.00000000e+00, 0.00000000e+00],
       [0.00000000e+00, 1.00000000e+00, 9.99799023e-01],
       [0.00000000e+00, 2.00000000e+00, 9.99416789e-01],
       ...,
       [1.20110000e+04, 1.20090000e+04, 4.19756312e-01],
       [1.20110000e+04, 1.20100000e+04, 2.26811434e-01],
       [1.20110000e+04, 1.20110000e+04, 0.00000000e+00]])

edge_index[:,2]

array([0.        , 0.99979902, 0.99941679, ..., 0.41975631, 0.22681143,
       0.        ])

Q. 그런데 밑에서 random으로 train하고 test로 나누게 되면.. wieght랑 edge를 어떻게 적용시키지?

edge: 같은 cc_num이면 edge=1, 다르면 edge=0

edge_index_list2 = []
for i in range(N):
    for j in range(N):
        if df50['cc_num'][i] != df50['cc_num'][j]:  
            edge = 0
        else:
            edge = 1
        edge_index_list2.append([i, j, edge])

np.save('edge_index_list2_50.npy', edge_index_list2)

loaded_data = np.load('edge_index_list2_50.npy')

edge_index_list2[:5]

[[0, 0, 1], [0, 1, 0], [0, 2, 0], [0, 3, 1], [0, 4, 0]]

edge_one = [(i, j) for i, j, edge in edge_index_list2 if edge == 1]
edge_one[:5]

[(0, 0), (0, 3), (0, 5), (0, 6), (0, 13)]

len(edge_one)

200706

edge_one_index = torch.tensor(edge_one, dtype=torch.long).t()

edge_one_index.shape

torch.Size([2, 200706])

edge_list2를 만든다 해도.. 밑에서 다시 적용하려면 index가 달라지면 못하ㅏ는거아닌가?

---

tr/test

df50_tr,df50_test = sklearn.model_selection.train_test_split(df50, random_state=42)

df50_tr.is_fraud.mean().round(5), df50_test.is_fraud.mean().round(5)

(0.49828, 0.50516)

df50_tr.shape, df50_test.shape

((9009, 23), (3003, 23))

train_mask = np.concatenate((np.full(9009, True), np.full(3003, False)))
test_mask = np.concatenate((np.full(9009, False), np.full(3003, True)))
print("Train Mask:", train_mask)
print("Test Mask:", test_mask)

Train Mask: [ True  True  True ... False False False]
Test Mask: [False False False ...  True  True  True]

train_mask.shape, test_mask.shape

((12012,), (12012,))

train_mask.sum(), test_mask.sum()

(9009, 3003)

여기서 tr/test를 나눠서 하는건 안될거같어.

---

data설정(x, edge_index, y)

x = df50['amt']

a = torch.tensor(x, dtype=torch.float)

a = a.reshape(-1,1)
a

tensor([[281.0600],
        [ 11.5200],
        [276.3100],
        ...,
        [  8.1200],
        [  3.5200],
        [ 84.1500]])

y = df50['is_fraud']

b = torch.tensor(y,dtype=torch.int64)

b

tensor([1, 1, 1,  ..., 0, 0, 0])

import torch_geometric

data = torch_geometric.data.Data(x=a, edge_index = edge_one_index, y=b, train_mask = train_mask, test_mask = test_mask)

data

Data(x=[12012, 1], edge_index=[2, 200706], y=[12012], train_mask=[12012], test_mask=[12012])

흠 .. 위의 edge_index에서. 각각의 w를 어떻게 연산해주려나

x랑 y의 순서를 무작위로 바꿔야하눈뎀. > 음 그럼 일단 edge_one_index이거부터 또 다싷..?

---

data설정 다시?

x = np.concatenate((np.array(df50_tr['amt']), np.array(df50_test['amt'])))

a = torch.tensor(x2, dtype=torch.float)

a = a.reshape(-1,1)
a

tensor([[921.2400],
        [698.2800],
        [220.5600],
        ...,
        [ 17.9700],
        [  7.5800],
        [824.9900]])

y = np.concatenate((np.array(df50_tr['is_fraud']), np.array(df50_test['is_fraud'])))

b = torch.tensor(y,dtype=torch.int64)

data = torch_geometric.data.Data(x=a, edge_index = edge_one_index, y=b, train_mask = train_mask, test_mask = test_mask)

바꿔버리면 ..

---

gnn

import torch
import torch.nn.functional as F
from torch_geometric.nn import GCNConv

class GCN(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = GCNConv(1, 16)
        self.conv2 = GCNConv(16,2)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

model = GCN()

model

GCN(
  (conv1): GCNConv(1, 16)
  (conv2): GCNConv(16, 2)
)

optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
model.train()

GCN(
  (conv1): GCNConv(1, 16)
  (conv2): GCNConv(16, 2)
)

---

for epoch in range(200):
    optimizer.zero_grad()
    out = model(data)
    loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
    loss.backward()
    optimizer.step()

model.eval()
pred = model(data).argmax(dim=1)
correct = (pred[data.test_mask] == data.y[data.test_mask]).sum()
acc = int(correct) / int(data.test_mask.sum())
print(f'Accuracy: {acc:.4f}')

Accuracy: 0.5758

out[data.train_mask]

tensor([[-0.7724, -0.6197],
        [-0.9459, -0.4916],
        [-0.9459, -0.4916],
        ...,
        [-0.7770, -0.6158],
        [-0.6621, -0.7252],
        [-0.6588, -0.7287]], grad_fn=<IndexBackward0>)

data.y[data.train_mask].sum()

tensor(4489)

data.test_mask.sum()

3003

correct/3003

tensor(0.5758)

---

---

model.eval()
pred = model(data).argmax(dim=1)
correct = (pred == data.y).sum() # 애큐러시는 test
acc = int(correct) / 9009
print(f'Accuracy: {acc:.4f}')

Accuracy: 0.9325

고려할 것(230810)2

• 현재 df50의 fraud 비율은 5:5 인데, 다른 비율을 가진 데이터로도 해보자

• GNN으로 돌려본 것과 다른 방법들과 비교를 해보자

• undersampling한 다른 데이터들과 비교해 볼 수 있을 듯(boost, logis, …)

• 9000/3000 데이터를 통해 합성 데이터를 만드는데, 12000개를 그대로 만드는 방법, 고객별로(cc_num) 합성 데이터를 만드는 방법, 똑같은 cc_num로 특이한 데이터가 있다면 normal데이터와 특이 데이터를 생각해서 돌리는 방법 등을 고려하자.

edge_index = np.array(edge_index_list)

edge_index.shape

(81162081, 3)

edge_index[:,2] = np.abs(edge_index[:,2])

theta = edge_index[:,2].mean()
theta

12230796.273867842

edge_index[:,2] = (np.exp(-edge_index[:,2]/theta)!=1) * np.exp(-edge_index[:,2]/theta)

edge_index

array([[0.00000000e+00, 0.00000000e+00, 0.00000000e+00],
       [0.00000000e+00, 1.00000000e+00, 1.90157975e-01],
       [0.00000000e+00, 2.00000000e+00, 9.79646259e-02],
       ...,
       [9.00800000e+03, 9.00600000e+03, 6.60662164e-01],
       [9.00800000e+03, 9.00700000e+03, 1.49150646e-01],
       [9.00800000e+03, 9.00800000e+03, 0.00000000e+00]])

eee = edge_index[:,:]

eee[:,1]

array([0.000e+00, 1.000e+00, 2.000e+00, ..., 9.006e+03, 9.007e+03,
       9.008e+03])

edge_index_list_updated = edge_index.tolist()

edge_index_list_updated[:5]

[[0.0, 0.0, 0.0],
 [0.0, 1.0, 0.19015797528259762],
 [0.0, 2.0, 0.09796462590589798],
 [0.0, 3.0, 0.1424157407389685],
 [0.0, 4.0, 0.11107338192969567]]

• cc_num로 그룹별로 묶자.

edge_index = np.array(edge_index_list)

edge_index.shape

(81162081, 3)

edge_index

array([[0.000e+00, 0.000e+00, 0.000e+00],
       [0.000e+00, 1.000e+00, 0.000e+00],
       [0.000e+00, 2.000e+00, 0.000e+00],
       ...,
       [9.008e+03, 9.006e+03, 0.000e+00],
       [9.008e+03, 9.007e+03, 0.000e+00],
       [9.008e+03, 9.008e+03, 0.000e+00]])

edge_index[:,2] = np.abs(edge_index[:,2])

theta = edge_index[:,2].mean()
theta

10988.585252761077

edge_index

array([[0.000e+00, 0.000e+00, 0.000e+00],
       [0.000e+00, 1.000e+00, 0.000e+00],
       [0.000e+00, 2.000e+00, 0.000e+00],
       ...,
       [9.008e+03, 9.006e+03, 0.000e+00],
       [9.008e+03, 9.007e+03, 0.000e+00],
       [9.008e+03, 9.008e+03, 0.000e+00]])

edge_index[:,2] = (np.exp(-edge_index[:,2]/theta)!=1) * np.exp(-edge_index[:,2]/theta)

edge_index

array([[0.000e+00, 0.000e+00, 0.000e+00],
       [0.000e+00, 1.000e+00, 0.000e+00],
       [0.000e+00, 2.000e+00, 0.000e+00],
       ...,
       [9.008e+03, 9.006e+03, 0.000e+00],
       [9.008e+03, 9.007e+03, 0.000e+00],
       [9.008e+03, 9.008e+03, 0.000e+00]])

edge_index_list_updated = edge_index.tolist()

np.array(edge_index_list_updated)[:,2].mean()

8.344409093328692e-05

mm = np.array(edge_index_list_updated)[:,2].mean()

edge_index_list_updated가 w

selected_edges = [(int(row[0]), int(row[1])) for row in edge_index_list_updated if row[2] > mm]

edge_index_selected = torch.tensor(selected_edges, dtype=torch.long).t()

edge_index_selected.shape

torch.Size([2, 28472])

edge_index

array([[0.000e+00, 0.000e+00, 0.000e+00],
       [0.000e+00, 1.000e+00, 0.000e+00],
       [0.000e+00, 2.000e+00, 0.000e+00],
       ...,
       [9.008e+03, 9.006e+03, 0.000e+00],
       [9.008e+03, 9.007e+03, 0.000e+00],
       [9.008e+03, 9.008e+03, 0.000e+00]])

- pyg lesson6

gconv = torch_geometric.nn.GCNConv(1,4)
gconv

GCNConv(1, 4)

gconv(data.x, data.edge_index)

tensor([[ 4.0225e+02,  2.5312e+02, -2.9747e+02, -1.6831e+02],
        [ 3.7246e+02,  2.3437e+02, -2.7543e+02, -1.5584e+02],
        [ 1.5695e+02,  9.8760e+01, -1.1606e+02, -6.5670e+01],
        ...,
        [ 2.5448e+02,  1.6013e+02, -1.8818e+02, -1.0648e+02],
        [ 5.4738e+02,  3.4444e+02, -4.0478e+02, -2.2903e+02],
        [ 1.1670e+00,  7.3434e-01, -8.6299e-01, -4.8830e-01]],
       grad_fn=<AddBackward0>)

list(gconv.parameters())

[Parameter containing:
 tensor([0., 0., 0., 0.], requires_grad=True),
 Parameter containing:
 tensor([[ 0.7116],
         [ 0.4478],
         [-0.5262],
         [-0.2977]], requires_grad=True)]

_,W = list(gconv.parameters())
W

Parameter containing:
tensor([[-0.6724],
        [ 0.7172],
        [-0.3185],
        [ 0.5363]], requires_grad=True)

- pyg lesson5

import torch
import torch.nn.functional as F
from torch_geometric.nn import GCNConv

class GCN(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = GCNConv(1, 16)
        self.conv2 = GCNConv(16,2)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

model = GCN()

model

GCN(
  (conv1): GCNConv(1, 16)
  (conv2): GCNConv(16, 2)
)

optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
model.train()

GCN(
  (conv1): GCNConv(1, 16)
  (conv2): GCNConv(16, 2)
)

out

tensor([[-1.8963e+02,  0.0000e+00],
        [-1.5192e+02,  0.0000e+00],
        [-5.3630e+01,  0.0000e+00],
        ...,
        [-3.0590e+02,  0.0000e+00],
        [-3.0298e+02,  0.0000e+00],
        [-1.3924e+00, -2.8567e-01]], grad_fn=<LogSoftmaxBackward0>)

data.y

tensor([1, 1, 0,  ..., 1, 1, 0])

for epoch in range(200):
    optimizer.zero_grad()
    out = model(data)
    loss = F.nll_loss(out, data.y)
    loss.backward()
    optimizer.step()

model.eval()
pred = model(data).argmax(dim=1)
correct = (pred == data.y).sum() # 애큐러시는 test
acc = int(correct) / 9009
print(f'Accuracy: {acc:.4f}')

Accuracy: 0.9633

fraud_mask = (data.y == 1)

model.eval()
pred = model(data).argmax(dim=1)
correct = (pred[fraud_mask] == data.y[fraud_mask]).sum() # 애큐러시는 test
acc = int(correct) / int(fraud_mask.sum())
print(f'recall: {acc:.4f}')

recall: 0.9619

• 위의 recall은 test가 없어서 train으로만 했던 거..!