bug fixed

Author: chenxj

README.md

@@ -19,9 +19,9 @@ A PyTorch implementation of DeepFM for CTR prediction problem.
 
 ## Reference
 
-- https://github.com/nzc/dnn_ctr
+- https://github.com/nzc/dnn_ctr.
 
-- https://github.com/PaddlePaddle/models/tree/develop/deep_fm
+- https://github.com/PaddlePaddle/models/tree/develop/deep_fm.
 
 - DeepFM: A Factorization-Machine based Neural Network for CTR         Prediction, Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He.
 

data/dataset.py

@@ -4,6 +4,8 @@
 import numpy as np
 import os
 
+continous_features = 13
+
 class CriteoDataset(Dataset):
     """
     Custom dataset class for Criteo dataset in order to use efficient 
@@ -34,13 +36,27 @@ def __init__(self, root, train=True):
     def __getitem__(self, idx):
         if self.train:
             dataI, targetI = self.train_data[idx, :], self.target[idx]
-            Xi = torch.from_numpy(dataI.astype(np.int32)).unsqueeze(-1)
-            Xv = torch.from_numpy(np.ones_like(dataI))
+            # index of continous features are zero
+            Xi_coutinous = np.zeros_like(dataI[:continous_features])
+            Xi_categorial = dataI[continous_features:]
+            Xi = torch.from_numpy(np.concatenate((Xi_coutinous, Xi_categorial)).astype(np.int32)).unsqueeze(-1)
+            
+            # value of categorial features are one (one hot features)
+            Xv_categorial = np.ones_like(dataI[continous_features:])
+            Xv_coutinous = dataI[:continous_features]
+            Xv = torch.from_numpy(np.concatenate((Xv_coutinous, Xv_categorial)).astype(np.int32))
             return Xi, Xv, targetI
         else:
             dataI = self.test_data.iloc[idx, :]
-            Xi = torch.from_numpy(dataI.astype(np.int32)).unsqueeze(-1)
-            Xv = torch.from_numpy(np.ones_like(dataI))
+            # index of continous features are one
+            Xi_coutinous = np.ones_like(dataI[:continous_features])
+            Xi_categorial = dataI[continous_features:]
+            Xi = torch.from_numpy(np.concatenate((Xi_coutinous, Xi_categorial)).astype(np.int32)).unsqueeze(-1)
+            
+            # value of categorial features are one (one hot features)
+            Xv_categorial = np.ones_like(dataI[continous_features:])
+            Xv_coutinous = dataI[:continous_features]
+            Xv = torch.from_numpy(np.concatenate((Xv_coutinous, Xv_categorial)).astype(np.int32))
             return Xi, Xv
 
     def __len__(self):

main.py

@@ -9,15 +9,15 @@
 from data.dataset import CriteoDataset
 
 # 900000 items for training, 10000 items for valid, of all 1000000 items
-Num_train = 900000
+Num_train = 9000
 
 # load data
 train_data = CriteoDataset('./data', train=True)
 loader_train = DataLoader(train_data, batch_size=100,
                           sampler=sampler.SubsetRandomSampler(range(Num_train)))
 val_data = CriteoDataset('./data', train=True)
 loader_val = DataLoader(val_data, batch_size=100,
-                        sampler=sampler.SubsetRandomSampler(range(Num_train, 1000000)))
+                        sampler=sampler.SubsetRandomSampler(range(Num_train, 10000)))
 
 feature_sizes = np.loadtxt('./data/feature_sizes.txt', delimiter=',')
 feature_sizes = [int(x) for x in feature_sizes]

model/DeepFM.py

@@ -27,12 +27,12 @@ class DeepFM(nn.Module):
     """
 
     def __init__(self, feature_sizes, embedding_size=4,
-                 hidden_dims=[32, 32], num_classes=10, dropout=[0.5, 0.5], 
+                 hidden_dims=[32, 32], num_classes=1, dropout=[0.5, 0.5], 
                  use_cuda=True, verbose=False):
         """
         Initialize a new network
 
-        Inputs:
+        Inputs: 
         - feature_size: A list of integer giving the size of features for each field.
         - embedding_size: An integer giving size of feature embedding.
         - hidden_dims: A list of integer giving the size of each hidden layer.
@@ -89,18 +89,10 @@ def forward(self, Xi, Xv):
         """
             fm part
         """
-        emb = self.fm_first_order_embeddings[20]
-        print(Xi.size())
-        for num in Xi[:, 20, :][0]:
-            if num > self.feature_sizes[20]:
-                print("index out")
 
         fm_first_order_emb_arr = [(torch.sum(emb(Xi[:, i, :]), 1).t() * Xv[:, i]).t() for i, emb in enumerate(self.fm_first_order_embeddings)]
-        # fm_first_order_emb_arr = [(emb(Xi[:, i]) * Xv[:, i])  for i, emb in enumerate(self.fm_first_order_embeddings)]
         fm_first_order = torch.cat(fm_first_order_emb_arr, 1)
-        # use 2xy = (x+y)^2 - x^2 - y^2 reduce calculation
         fm_second_order_emb_arr = [(torch.sum(emb(Xi[:, i, :]), 1).t() * Xv[:, i]).t() for i, emb in enumerate(self.fm_second_order_embeddings)]
-        # fm_second_order_emb_arr = [(emb(Xi[:, i]) * Xv[:, i]) for i, emb in enumerate(self.fm_second_order_embeddings)]
         fm_sum_second_order_emb = sum(fm_second_order_emb_arr)
         fm_sum_second_order_emb_square = fm_sum_second_order_emb * \
             fm_sum_second_order_emb  # (x+y)^2
@@ -115,7 +107,7 @@ def forward(self, Xi, Xv):
         """
         deep_emb = torch.cat(fm_second_order_emb_arr, 1)
         deep_out = deep_emb
-        for i in range(1, self.hidden_dims + 1):
+        for i in range(1, len(self.hidden_dims) + 1):
             deep_out = getattr(self, 'linear_' + str(i))(deep_out)
             deep_out = getattr(self, 'batchNorm_' + str(i))(deep_out)
             deep_out = getattr(self, 'dropout_' + str(i))(deep_out)
@@ -126,7 +118,7 @@ def forward(self, Xi, Xv):
                     torch.sum(fm_second_order, 1) + torch.sum(deep_out, 1) + self.bias
         return total_sum
 
-    def fit(self, loader_train, loader_val, optimizer, epochs=1, verbose=False, print_every=100):
+    def fit(self, loader_train, loader_val, optimizer, epochs=100, verbose=False, print_every=100):
         """
         Training a model and valid accuracy.
 
@@ -148,7 +140,7 @@ def fit(self, loader_train, loader_val, optimizer, epochs=1, verbose=False, prin
             for t, (xi, xv, y) in enumerate(loader_train):
                 xi = xi.to(device=self.device, dtype=self.dtype)
                 xv = xv.to(device=self.device, dtype=torch.float)
-                y = y.to(device=self.device, dtype=self.dtype)
+                y = y.to(device=self.device, dtype=torch.float)
 
                 total = model(xi, xv)
                 loss = criterion(total, y)
@@ -172,8 +164,8 @@ def check_accuracy(self, loader, model):
         with torch.no_grad():
             for xi, xv, y in loader:
                 xi = xi.to(device=self.device, dtype=self.dtype)  # move to device, e.g. GPU
-                xv = xv.to(device=self.device, dtype=self.dtype)
-                y = y.to(device=self.device, dtype=self.dtype)
+                xv = xv.to(device=self.device, dtype=torch.float)
+                y = y.to(device=self.device, dtype=torch.bool)
                 total = model(xi, xv)
                 preds = (F.sigmoid(total) > 0.5)
                 num_correct += (preds == y).sum()

utils/dataPreprocess.py

@@ -83,7 +83,7 @@ def gen(self, idx, val):
 # @click.command("preprocess")
 # @click.option("--datadir", type=str, help="Path to raw criteo dataset")
 # @click.option("--outdir", type=str, help="Path to save the processed data")
-def preprocess(datadir, outdir):
+def preprocess(datadir, outdir, num_train_sample = 10000, num_test_sample = 10000):
     """
     All the 13 integer features are normalzied to continous values and these
     continous features are combined into one vecotr with dimension 13.
@@ -98,11 +98,6 @@ def preprocess(datadir, outdir):
         os.path.join(datadir, 'train.txt'), categorial_features, cutoff=200)
 
     dict_sizes = dicts.dicts_sizes()
-    categorial_feature_offset = [0]
-    for i in range(1, len(categorial_features)):
-        offset = categorial_feature_offset[i - 1] + dict_sizes[i - 1]
-        categorial_feature_offset.append(offset)
-
     with open(os.path.join(outdir, 'feature_sizes.txt'), 'w') as feature_sizes:
         sizes = [1] * len(continous_features) + dict_sizes
         sizes = [str(i) for i in sizes]
@@ -113,7 +108,7 @@ def preprocess(datadir, outdir):
     # Saving the data used for training.
     with open(os.path.join(outdir, 'train.txt'), 'w') as out_train:
         with open(os.path.join(datadir, 'train.txt'), 'r') as f:
-            for line in f:
+            for line in f.readlines()[:num_train_sample]:
                 features = line.rstrip('\n').split('\t')
 
                 continous_vals = []
@@ -124,7 +119,7 @@ def preprocess(datadir, outdir):
                 categorial_vals = []
                 for i in range(0, len(categorial_features)):
                     val = dicts.gen(i, features[categorial_features[
-                        i]]) + categorial_feature_offset[i]
+                        i]])
                     categorial_vals.append(str(val))
 
                 continous_vals = ','.join(continous_vals)
@@ -135,7 +130,7 @@ def preprocess(datadir, outdir):
 
     with open(os.path.join(outdir, 'test.txt'), 'w') as out:
         with open(os.path.join(datadir, 'test.txt'), 'r') as f:
-            for line in f:
+            for line in f.readlines()[:num_test_sample]:
                 features = line.rstrip('\n').split('\t')
 
                 continous_vals = []
@@ -146,7 +141,7 @@ def preprocess(datadir, outdir):
                 categorial_vals = []
                 for i in range(0, len(categorial_features)):
                     val = dicts.gen(i, features[categorial_features[
-                        i] - 1]) + categorial_feature_offset[i]
+                        i] - 1])
                     categorial_vals.append(str(val))
 
                 continous_vals = ','.join(continous_vals)