main
parent
d881f98047
commit
c2c081e0df
7 changed files with 352 additions and 0 deletions
@ -0,0 +1,94 @@ |
|||||||
|
import os |
||||||
|
import random |
||||||
|
|
||||||
|
import numpy as np |
||||||
|
import scipy.sparse as sp |
||||||
|
|
||||||
|
from src import config |
||||||
|
|
||||||
|
|
||||||
|
def load_luo_data(dataset): |
||||||
|
dp = np.loadtxt('../../data/RawData/luo/mat_drug_protein.txt'.format(dataset), dtype=int) |
||||||
|
dd = np.loadtxt('../../data/RawData/luo/mat_drug_drug.txt'.format(dataset), dtype=int) |
||||||
|
pp = np.loadtxt('../../data/RawData/luo/mat_protein_protein.txt'.format(dataset), dtype=int) |
||||||
|
adj = np.vstack((np.hstack((dd, dp)), np.hstack((dp.T, pp)))) |
||||||
|
return sp.csr_matrix(adj + sp.eye(adj.shape[0])), dd.shape[0] |
||||||
|
|
||||||
|
|
||||||
|
def load_yam_data(dataset): |
||||||
|
dp = np.loadtxt('../../data/RawData/Yamanishi/{}_admat_dgc.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.int).T |
||||||
|
dd = np.loadtxt('../../data/RawData/Yamanishi/{}_simmat_dc.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.float) |
||||||
|
pp = np.loadtxt('../../data/RawData/Yamanishi/{}_simmat_dg.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.float) |
||||||
|
dd = np.where(dd < 0.5, 0, 1) |
||||||
|
pp = np.where(pp < 0.5, 0, 1) |
||||||
|
adj = np.vstack((np.hstack((dd, dp)), np.hstack((dp.T, pp)))) |
||||||
|
return sp.csr_matrix(adj), dd.shape[0] |
||||||
|
|
||||||
|
|
||||||
|
def is_symmetry(adj): |
||||||
|
for i in range(adj.shape[0]): |
||||||
|
for j in range(adj.shape[1]): |
||||||
|
if adj[i][j] != adj[j][i]: |
||||||
|
return False |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
def is_1_diag(adj): |
||||||
|
if sum(np.diagonal(adj)) != adj.shape[0]: |
||||||
|
return False |
||||||
|
return True |
||||||
|
|
||||||
|
|
||||||
|
def change_unbalanced(adj, percent, dp_line, dataset): |
||||||
|
""" |
||||||
|
note: percent控制屏蔽掉的节点所占的百分比 |
||||||
|
:param adj: |
||||||
|
:param percent: |
||||||
|
:return: 返回去除部分已知关联的邻接矩阵 |
||||||
|
""" |
||||||
|
# 判断是否对称 |
||||||
|
# assert is_symmetry(adj.A) |
||||||
|
adj = adj - sp.dia_matrix((adj.diagonal()[np.newaxis, :], [0]), shape=adj.shape) + sp.eye(adj.shape[0]) |
||||||
|
# 判断对角线是否全为1 |
||||||
|
assert is_1_diag(adj.A) |
||||||
|
adj = (sp.triu(adj) + sp.triu(adj).T - sp.eye(adj.shape[0])).A |
||||||
|
|
||||||
|
row = list(range(0, dp_line)) |
||||||
|
col = list(range(dp_line, adj.shape[0])) |
||||||
|
|
||||||
|
idx = [] |
||||||
|
for i in row: |
||||||
|
for j in col: |
||||||
|
if i != j and adj[i][j] == 1: |
||||||
|
idx.append((i, j)) |
||||||
|
num = int(np.floor(percent * len(idx))) |
||||||
|
count = 0 |
||||||
|
# random.seed(config.seed) |
||||||
|
while count < num: |
||||||
|
row, col = random.choice(idx) |
||||||
|
idx.remove((row, col)) |
||||||
|
adj[row][col] = 0 |
||||||
|
adj[col][row] = 0 |
||||||
|
count += 1 |
||||||
|
|
||||||
|
# idx = [] |
||||||
|
# for i in range(adj.shape[0]): |
||||||
|
# for j in range(i + 1, adj.shape[0]): |
||||||
|
# if adj[i][j] == 1: |
||||||
|
# idx.append((i, j)) |
||||||
|
# num = int(np.floor(percent * len(idx))) |
||||||
|
# count = 0 |
||||||
|
# # random.seed(config.seed) |
||||||
|
# while count < num: |
||||||
|
# row, col = random.choice(idx) |
||||||
|
# idx.remove((row, col)) |
||||||
|
# adj[row][col] = 0 |
||||||
|
# adj[col][row] = 0 |
||||||
|
# count += 1 |
||||||
|
|
||||||
|
# 保存改变不平衡性后新的dp |
||||||
|
new_dp = adj[0:dp_line, dp_line:] |
||||||
|
# if not os.path.exists('../../data/partitioned_data/{0}/feature'.format(dataset)): |
||||||
|
# os.mkdir('../../data/partitioned_data/{0}/feature'.format(dataset)) |
||||||
|
# np.savetxt('../../data/partitioned_data/{0}/feature/{0}_new_admat_dgc.txt'.format(dataset), new_dp, fmt='%d', delimiter='\t') |
||||||
|
return sp.csr_matrix(adj.astype(np.int)) |
@ -0,0 +1,88 @@ |
|||||||
|
import os |
||||||
|
import pickle |
||||||
|
|
||||||
|
import numpy as np |
||||||
|
|
||||||
|
from src import config |
||||||
|
import scipy.sparse as sp |
||||||
|
|
||||||
|
from load_data import load_yam_data, change_unbalanced, load_luo_data |
||||||
|
from utils import divide_vgae_datasets, sparse_to_tuple, divide_datasets |
||||||
|
|
||||||
|
for dataset in config.datasets: |
||||||
|
g = os.walk(r"../../data/partitioned_data/{}".format(dataset)) |
||||||
|
for path, dir_list, file_list in g: |
||||||
|
for file_name in file_list: |
||||||
|
os.remove(os.path.join(path, file_name)) |
||||||
|
print("清除缓存完成!") |
||||||
|
|
||||||
|
# Load data 得到一个邻接矩阵,双向边 |
||||||
|
if dataset == 'luo': |
||||||
|
adj, dp_line = load_luo_data(dataset) |
||||||
|
else: |
||||||
|
adj, dp_line = load_yam_data(dataset) |
||||||
|
|
||||||
|
if not os.path.exists("../../data/partitioned_data"): |
||||||
|
os.mkdir("../../data/partitioned_data") |
||||||
|
if not os.path.exists("../../data/partitioned_data/{}".format(dataset)): |
||||||
|
os.mkdir("../../data/partitioned_data/{}".format(dataset)) |
||||||
|
if not os.path.exists("../../data/partitioned_data/{}/orig".format(dataset)): |
||||||
|
os.mkdir("../../data/partitioned_data/{}/orig/".format(dataset)) |
||||||
|
np.savetxt("../../data/partitioned_data/{}/orig/dp_line.txt".format(dataset), np.array([dataset, str(dp_line)]), fmt='%s') |
||||||
|
|
||||||
|
# 获得不同不平衡性的数据 |
||||||
|
adj = change_unbalanced(adj, config.percent, dp_line, dataset) |
||||||
|
|
||||||
|
# Store original adjacency matrix (without diagonal entries) for later 保存原始邻接矩阵(不含对角线项)以备后用 |
||||||
|
adj_orig = adj |
||||||
|
adj_orig = adj_orig - sp.dia_matrix((adj_orig.diagonal()[np.newaxis, :], [0]), shape=adj_orig.shape) # 假设对角线有元素,去除对角线 |
||||||
|
adj_orig.eliminate_zeros() # 假设有0,移除矩阵中的0 |
||||||
|
path = "../../data/partitioned_data/{}/orig/".format(dataset) |
||||||
|
if not os.path.exists(path): |
||||||
|
os.makedirs(path) |
||||||
|
pickle.dump(adj_orig, open(path + dataset + "_adj_orig.pkl", 'wb')) |
||||||
|
np.savetxt(path + dataset + "_adj_orig.txt", adj_orig.A, fmt='%d') |
||||||
|
|
||||||
|
# 为获取嵌入划分数据, 划分数据集, 并记录边 |
||||||
|
for i in range(10): |
||||||
|
# Remove diagonal elements # 删除对角线元素 |
||||||
|
adj = adj - sp.dia_matrix((adj.diagonal()[np.newaxis, :], [0]), shape=adj.shape) # 梅开二度 |
||||||
|
adj.eliminate_zeros() |
||||||
|
# Check that diag is zero: # 检查diag是否为零: |
||||||
|
assert np.diag(adj.todense()).sum() == 0 |
||||||
|
|
||||||
|
# 为graphgan划分数据 |
||||||
|
g_adj = adj[0:dp_line, dp_line:] |
||||||
|
g_edges = sparse_to_tuple(g_adj)[0] |
||||||
|
g_num_test = int(np.floor(g_edges.shape[0] / 10.)) # np.floor()是向下取整。测试集10分之一,训练集20分之一 |
||||||
|
g_num_val = int(np.floor(g_edges.shape[0] / 20.)) |
||||||
|
|
||||||
|
adj_pd, train_edges, test_edges, test_edges_false = divide_datasets(g_adj, g_edges, g_num_test, i, dp_line) |
||||||
|
adj[0:dp_line, dp_line:] = adj_pd |
||||||
|
|
||||||
|
# 将训练集分给vgae |
||||||
|
edges = sparse_to_tuple(sp.triu(adj))[0] |
||||||
|
edges_all = sparse_to_tuple(adj)[0] # 将邻接矩阵转换成三元组,然后只取坐标,即所有的边 |
||||||
|
num_test = int(np.floor(edges.shape[0] / 10.)) # np.floor()是向下取整。测试集10分之一,训练集20分之一 |
||||||
|
num_val = int(np.floor(edges.shape[0] / 20.)) |
||||||
|
|
||||||
|
adj_train, vgae_train_edges, vgae_test_edges, vgae_test_edges_false = divide_vgae_datasets(adj, edges, edges_all, num_test, num_val, |
||||||
|
i) # val_edges, val_edges_false, |
||||||
|
|
||||||
|
# 保存划分好的数据 |
||||||
|
path = "../../data/partitioned_data/{}/{}fold/".format(dataset, i) |
||||||
|
if not os.path.exists(path): |
||||||
|
os.makedirs(path) |
||||||
|
|
||||||
|
pickle.dump(adj_train, open(path + dataset + "_adj_train.pkl", 'wb')) |
||||||
|
|
||||||
|
np.savetxt(path + dataset + "_vgae_train.txt", vgae_train_edges, fmt='%d') |
||||||
|
np.savetxt(path + dataset + "_vgae_test.txt", vgae_test_edges, fmt='%d') |
||||||
|
np.savetxt(path + dataset + "_vgae_test_neg.txt", vgae_test_edges_false, fmt='%d') |
||||||
|
|
||||||
|
np.savetxt(path + dataset + "_train.txt", vgae_train_edges, fmt='%d') |
||||||
|
np.savetxt(path + dataset + "_pd_train.txt", train_edges, fmt='%d') |
||||||
|
np.savetxt(path + dataset + "_test.txt", test_edges, fmt='%d') |
||||||
|
np.savetxt(path + dataset + "_test_neg.txt", test_edges_false, fmt='%d') |
||||||
|
|
||||||
|
print("OK") |
@ -0,0 +1,130 @@ |
|||||||
|
import numpy as np |
||||||
|
import scipy.sparse as sp |
||||||
|
|
||||||
|
from src import config |
||||||
|
|
||||||
|
|
||||||
|
def sparse_to_tuple(sparse_mx): |
||||||
|
if not sp.isspmatrix_coo(sparse_mx): |
||||||
|
sparse_mx = sparse_mx.tocoo() |
||||||
|
coords = np.vstack((sparse_mx.row, sparse_mx.col)).transpose() |
||||||
|
values = sparse_mx.data |
||||||
|
shape = sparse_mx.shape |
||||||
|
return coords, values, shape |
||||||
|
|
||||||
|
|
||||||
|
def divide_vgae_datasets(adj, edges, edges_all, num_test, num_val, i): |
||||||
|
# 构建具有10%正向链接的测试集的函数 |
||||||
|
# 注:拆分是随机的,结果可能与论文中报告的数字略有偏差。 |
||||||
|
|
||||||
|
if i == 9: |
||||||
|
start_test = num_test * i |
||||||
|
end_test = edges.shape[0] |
||||||
|
start_val = 0 |
||||||
|
end_val = num_val |
||||||
|
else: |
||||||
|
start_test = num_test * i |
||||||
|
end_test = num_test * (i + 1) |
||||||
|
start_val = end_test |
||||||
|
end_val = end_test + num_val |
||||||
|
|
||||||
|
all_edge_idx = list(range(edges.shape[0])) |
||||||
|
np.random.seed(config.seed) |
||||||
|
np.random.shuffle(edges) |
||||||
|
# val_edge_idx = all_edge_idx[start_val:end_val] |
||||||
|
test_edge_idx = all_edge_idx[start_test:end_test] |
||||||
|
test_edges = edges[test_edge_idx] |
||||||
|
# val_edges = edges[val_edge_idx] |
||||||
|
train_edges = np.delete(edges, np.hstack([test_edge_idx]), axis=0) # , val_edge_idx |
||||||
|
|
||||||
|
def ismember(a: list, b, tol=5): |
||||||
|
rows_close = np.all(np.round(a - b[:, None], tol) == 0, axis=-1) |
||||||
|
return np.any(rows_close) |
||||||
|
|
||||||
|
test_edges_false = [] |
||||||
|
while len(test_edges_false) < len(test_edges): |
||||||
|
idx_i = np.random.randint(0, adj.shape[0]) # 随机生成横坐标 |
||||||
|
idx_j = np.random.randint(0, adj.shape[0]) # 随机生成纵坐标 |
||||||
|
if idx_i == idx_j: # 对角线的不要 |
||||||
|
continue |
||||||
|
if ismember([idx_i, idx_j], edges_all): # 是已知边不要 |
||||||
|
continue |
||||||
|
if test_edges_false: # 已选负边不要,a-b或b-a有一个是都不要 |
||||||
|
if ismember([idx_j, idx_i], np.array(test_edges_false)): |
||||||
|
continue |
||||||
|
if ismember([idx_i, idx_j], np.array(test_edges_false)): |
||||||
|
continue |
||||||
|
test_edges_false.append([idx_i, idx_j]) |
||||||
|
|
||||||
|
# val_edges_false = [] |
||||||
|
# while len(val_edges_false) < len(val_edges): |
||||||
|
# idx_i = np.random.randint(0, adj.shape[0]) |
||||||
|
# idx_j = np.random.randint(0, adj.shape[0]) |
||||||
|
# if idx_i == idx_j: # 对角线不要 |
||||||
|
# continue |
||||||
|
# if ismember([idx_i, idx_j], edges_all): # 是已知边不要 |
||||||
|
# continue |
||||||
|
# if val_edges_false: |
||||||
|
# if ismember([idx_j, idx_i], np.array(val_edges_false)): |
||||||
|
# continue |
||||||
|
# if ismember([idx_i, idx_j], np.array(val_edges_false)): |
||||||
|
# continue |
||||||
|
# val_edges_false.append([idx_i, idx_j]) |
||||||
|
|
||||||
|
assert ~ismember(test_edges_false, edges_all) |
||||||
|
# assert ~ismember(val_edges_false, edges_all) |
||||||
|
# assert ~ismember(val_edges, train_edges) |
||||||
|
assert ~ismember(test_edges, train_edges) |
||||||
|
# assert ~ismember(val_edges, test_edges) |
||||||
|
|
||||||
|
# Re-build adj matrix 重建邻接矩阵 |
||||||
|
adj_train = sp.csr_matrix((np.ones(train_edges.shape[0]), (train_edges[:, 0], train_edges[:, 1])), shape=adj.shape) |
||||||
|
adj_train = adj_train + adj_train.T # 因为train_edges是单向的,所以把它变成对称的 |
||||||
|
|
||||||
|
# NOTE: these edge lists only contain single direction of edge! 注意:这些边列表只包含边的单一方向! |
||||||
|
return adj, train_edges, test_edges, np.array(test_edges_false) # , val_edges, np.array(val_edges_false) |
||||||
|
|
||||||
|
|
||||||
|
def divide_datasets(adj, edges, num_test, i, dp_line): |
||||||
|
if i == 9: |
||||||
|
start_test = num_test * i |
||||||
|
end_test = edges.shape[0] |
||||||
|
else: |
||||||
|
start_test = num_test * i |
||||||
|
end_test = num_test * (i + 1) |
||||||
|
|
||||||
|
all_edge_idx = list(range(edges.shape[0])) |
||||||
|
np.random.seed(config.seed) |
||||||
|
np.random.shuffle(edges) |
||||||
|
test_edge_idx = all_edge_idx[start_test:end_test] |
||||||
|
test_edges = edges[test_edge_idx] |
||||||
|
train_edges = np.delete(edges, np.hstack([test_edge_idx]), axis=0) # , val_edge_idx |
||||||
|
|
||||||
|
def ismember(a: list, b, tol=5): |
||||||
|
rows_close = np.all(np.round(a - b[:, None], tol) == 0, axis=-1) |
||||||
|
return np.any(rows_close) |
||||||
|
|
||||||
|
test_edges_false = [] |
||||||
|
while len(test_edges_false) < len(test_edges): |
||||||
|
idx_i = np.random.randint(0, adj.shape[0]) # 随机生成横坐标 |
||||||
|
idx_j = np.random.randint(0, adj.shape[1]) # 随机生成纵坐标 |
||||||
|
if idx_i == idx_j: # 自身不要 |
||||||
|
continue |
||||||
|
if ismember([idx_i, idx_j], edges): # 是已知边不要 |
||||||
|
continue |
||||||
|
test_edges_false.append([idx_i, idx_j]) |
||||||
|
|
||||||
|
adj_pd = sp.csr_matrix((np.ones(train_edges.shape[0]), (train_edges[:, 0], train_edges[:, 1])), shape=adj.shape) |
||||||
|
|
||||||
|
# 把列索引编号加上dp_line |
||||||
|
def add_index(edges): |
||||||
|
edges = np.array(edges) |
||||||
|
colu = edges[:, 1] + dp_line |
||||||
|
edges[:, 1] = colu |
||||||
|
return edges |
||||||
|
|
||||||
|
train_edges = add_index(train_edges) |
||||||
|
test_edges = add_index(test_edges) |
||||||
|
test_edges_false = add_index(test_edges_false) |
||||||
|
|
||||||
|
return adj_pd, train_edges, test_edges, test_edges_false |
@ -0,0 +1,2 @@ |
|||||||
|
from __future__ import print_function |
||||||
|
from __future__ import division |
@ -0,0 +1,18 @@ |
|||||||
|
import numpy as np |
||||||
|
import scipy.sparse as sp |
||||||
|
|
||||||
|
|
||||||
|
def load_yam_feature(dataset): |
||||||
|
dp = np.loadtxt('../../data/RawData/Yamanishi/{}_admat_dgc.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.float).T |
||||||
|
dd = np.loadtxt('../../data/RawData/Yamanishi/{}_simmat_dc.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.float) |
||||||
|
pp = np.loadtxt('../../data/RawData/Yamanishi/{}_simmat_dg.txt'.format(dataset), dtype=str, delimiter='\t')[1:, 1:].astype(np.float) |
||||||
|
feature = np.vstack((np.hstack((dd, dp)), np.hstack((dp.T, pp)))) |
||||||
|
return sp.lil_matrix(feature) |
||||||
|
|
||||||
|
|
||||||
|
def load_luo_feature(dataset): |
||||||
|
dp = np.loadtxt('../../data/RawData/luo/mat_drug_protein.txt'.format(dataset), dtype=float) |
||||||
|
dd = np.loadtxt('../../data/RawData/luo/Similarity_Matrix_Drugs.txt'.format(dataset), dtype=float) |
||||||
|
pp = np.loadtxt('../../data/RawData/luo/Similarity_Matrix_Proteins.txt'.format(dataset), dtype=float) / 100 |
||||||
|
feature = np.vstack((np.hstack((dd, dp)), np.hstack((dp.T, pp)))) |
||||||
|
return sp.lil_matrix(feature) |
@ -0,0 +1,20 @@ |
|||||||
|
import os |
||||||
|
import pickle |
||||||
|
|
||||||
|
from src import config |
||||||
|
from src.p2_preprocessing_feature.load_feature import load_yam_feature, load_luo_feature |
||||||
|
|
||||||
|
for dataset in config.datasets: |
||||||
|
# feature: lil_matrix |
||||||
|
if dataset == 'luo': |
||||||
|
feature = load_luo_feature(dataset) |
||||||
|
else: |
||||||
|
feature = load_yam_feature(dataset) |
||||||
|
|
||||||
|
# 保存特征 |
||||||
|
path = "../../data/partitioned_data/{}/feature/".format(dataset) |
||||||
|
if not os.path.exists(path): |
||||||
|
os.makedirs(path) |
||||||
|
pickle.dump(feature, open(path + dataset + "_feature.pkl", 'wb')) |
||||||
|
|
||||||
|
print("ok") |
Loading…
Reference in new issue