refactor

Author: lindsey98

Influence_function/EIF_utils.py

@@ -6,37 +6,6 @@
 import torch.nn.functional as F
 import math
 
-# @torch.no_grad()
-# def calc_loss_train_relabel(model, dl, relabel_candidate, criterion, indices=None):
-#
-#     l_all = {}
-#     model.eval()
-#     for ct, (x, t, ind) in enumerate(dl):
-#         torch.cuda.empty_cache()
-#         if ind.item() in indices:
-#             y = relabel_candidate[ind.item()]
-#             x = x.expand(len(y), x.size()[1], x.size()[2], x.size()[3])
-#             m = model(x)
-#             l = criterion.debug(m, None, y) # (nb_classes, )
-#             l_all[ind.item()] = l.detach().cpu().numpy()
-#             pass
-#     l_final = []
-#     for ind in indices:
-#         l_final.append(l_all[ind])
-#     l_final = np.asarray(l_final)
-#     return l_final # (N, nb_classes)
-#
-# def loss_change_train_relabel(model, criterion, dl_tr, relabel_candidate, params_prev, params_cur, indices):
-#
-#     weight_orig = model.module[-1].weight.data # cache original parameters
-#     model.module[-1].weight.data = params_prev
-#     l_prev = calc_loss_train_relabel(model, dl_tr, relabel_candidate, criterion, indices) # (N, nb_classes)
-#
-#     model.module[-1].weight.data = params_cur
-#     l_cur = calc_loss_train_relabel(model, dl_tr, relabel_candidate, criterion, indices) # (N, nb_classes)
-#
-#     model.module[-1].weight.data = weight_orig # dont forget to revise the weights back to the original
-#     return l_prev, l_cur
 
 @torch.no_grad()
 def calc_loss_train(model, dl, criterion, indices=None):
@@ -45,7 +14,6 @@ def calc_loss_train(model, dl, criterion, indices=None):
     '''
     l = []
     model.eval()
-
     for ct, (x, t, _) in tqdm(enumerate(dl)):
         x, t = x.cuda(), t.cuda()
         m = model(x)
@@ -72,7 +40,7 @@ def loss_change_train(model, criterion, dl_tr, params_prev, params_cur):
 
 def calc_inter_dist_pair(feat_cls1, feat_cls2):
     '''
-        Calculate d(confusion pair)
+        Calculate d(p_c)
     '''
     feat_cls1 = F.normalize(feat_cls1, p=2, dim=-1) # L2 normalization
     feat_cls2 = F.normalize(feat_cls2, p=2, dim=-1)
@@ -86,15 +54,14 @@ def calc_inter_dist_pair(feat_cls1, feat_cls2):
 
 def grad_confusion_pair(model, all_features, wrong_indices, confusion_indices):
     '''
-        Calculate  \partial d(confusion pair) / \partial theta
+        Calculate  \triangle d(p_c) / \triangle theta
     '''
     cls_features = all_features[wrong_indices]
     confuse_cls_features = all_features[confusion_indices]
 
     model.zero_grad()
     model.eval()
-    cls_features = cls_features.cuda()
-    confuse_cls_features = confuse_cls_features.cuda()
+    cls_features, confuse_cls_features = cls_features.cuda(), confuse_cls_features.cuda()
 
     feature1 = model.module[-1](cls_features)  # (N', 512)
     feature2 = model.module[-1](confuse_cls_features)  # (N', 512)
@@ -110,7 +77,7 @@ def grad_confusion_pair(model, all_features, wrong_indices, confusion_indices):
 def grad_confusion(model, all_features, cls, confusion_classes,
                    pred, label, nn_indices):
     '''
-        Calculate  \partial avg{d(confusion pair)} / \partial theta
+        Calculate  \triangle Avg{d(p_c)} / \triangle theta
     '''
     pred = pred.detach().cpu().numpy()
     label = label.detach().cpu().numpy()

Influence_function/IF_utils.py

@@ -59,7 +59,6 @@ def inverse_hessian_product(model, criterion, v, dl_tr,
         hv = hessian_vector_product(loss, params, cur_estimate) # get hvp
         # Inverse Hessian product Update: v + (I - Hessian_at_x) * cur_estimate
         cur_estimate = [_v + (1 - damping) * _h_e - _hv.detach().cpu() / scale for _v, _h_e, _hv in zip(v, cur_estimate, hv)]
-        pass
 
     inverse_hvp = [b.detach().cpu() / scale for b in cur_estimate] # "In the loop, we scale the Hessian down by scale, which means that the estimate of the inverse Hessian-vector product will be scaled up by scale. The last division corrects for this scaling."
     return inverse_hvp # I didn't divide it by number of recursions

Influence_function/influence_function.py

@@ -66,100 +66,28 @@ def getNN_indices(self, embedding, label):
 
         return nn_indices, nn_label, nn_indices_same_cls
 
-    def vis_pairs(self, wrong_indices, confuse_indices, wrong_samecls_indices,
-                  dl, base_dir='Grad_Test'):
-        '''Visualize all confusion pairs'''
-        assert len(wrong_indices) == len(confuse_indices)
-        assert len(wrong_indices) == len(wrong_samecls_indices)
-
-        os.makedirs('./{}/{}'.format(base_dir, self.dataset_name), exist_ok=True)
-        model_copy = self._load_model()
-        model_copy.eval()
-
-        for ind1, ind2, ind3 in zip(wrong_indices, confuse_indices, wrong_samecls_indices):
-            # cam_extractor1 = GradCAMCustomize(model_copy, target_layer=model_copy.module[0].base.layer4)  # to last layer
-            # cam_extractor2 = GradCAMCustomize(model_copy, target_layer=model_copy.module[0].base.layer4)  # to last layer
-
-            # Get the two embeddings first
-            img1 = to_pil_image(read_image(dl.dataset.im_paths[ind1]))
-            img2 = to_pil_image(read_image(dl.dataset.im_paths[ind2]))
-            img3 = to_pil_image(read_image(dl.dataset.im_paths[ind3]))
-
-            # cam_extractor1._hooks_enabled = True
-            # model_copy.zero_grad()
-            # emb1 = model_copy(dl.dataset.__getitem__(ind1)[0].unsqueeze(0).cuda())
-            # emb2 = model_copy(dl.dataset.__getitem__(ind2)[0].unsqueeze(0).cuda())
-            # activation_map2 = cam_extractor1(torch.dot(emb1.detach().squeeze(0), emb2.squeeze(0)))
-            # result2, _ = overlay_mask(img2, to_pil_image(activation_map2[0].detach().cpu(), mode='F'), alpha=0.5)
-            #
-            # cam_extractor2._hooks_enabled = True
-            # model_copy.zero_grad()
-            # emb1 = model_copy(dl.dataset.__getitem__(ind1)[0].unsqueeze(0).cuda())
-            # emb3 = model_copy(dl.dataset.__getitem__(ind3)[0].unsqueeze(0).cuda())
-            # activation_map3 = cam_extractor2(torch.dot(emb1.detach().squeeze(0), emb3.squeeze(0)))
-            # result3, _ = overlay_mask(img3, to_pil_image(activation_map3[0].detach().cpu(), mode='F'), alpha=0.5)
-
-            # Display it
-            fig = plt.figure()
-            fig.subplots_adjust(top=0.8)
-
-            ax = fig.add_subplot(1, 3, 1)
-            ax.imshow(img1)
-            ax.title.set_text('Ind = {} \n Class = {}'.format(ind1, dl.dataset.ys[ind1]))
-            plt.axis('off')
-
-            ax = fig.add_subplot(1, 3, 2)
-            ax.imshow(img2)
-            ax.title.set_text('Ind = {} \n Class = {}'.format(ind2, dl.dataset.ys[ind2]))
-            plt.axis('off')
-
-            ax = fig.add_subplot(1, 3, 3)
-            ax.imshow(img3)
-            ax.title.set_text('Ind = {} \n Class = {}'.format(ind3, dl.dataset.ys[ind3]))
-            plt.axis('off')
-
-            plt.savefig('./{}/{}/{}_{}.png'.format(base_dir, self.dataset_name,
-                                                   ind1, ind2))
-            plt.close()
-
-    def calc_relabel_dict(self, lookat_harmful, relabel_method,
-                          harmful_indices, helpful_indices, train_nn_indices, train_nn_indices_same_cls,
+    def calc_relabel_dict(self, lookat_harmful,
+                          harmful_indices, helpful_indices,
                           base_dir, pair_ind1, pair_ind2):
 
         assert isinstance(lookat_harmful, bool)
-        assert relabel_method in ['hard', 'soft_knn']
         if lookat_harmful:
             top_indices = harmful_indices  # top_harmful_indices = influence_values.argsort()[-50:]
         else:
             top_indices = helpful_indices
-        top_nn_indices = train_nn_indices[top_indices]
-        top_nn_samecls_indices = train_nn_indices_same_cls[top_indices]
 
-        if relabel_method == 'hard': # relabel as its 1st NN
-            relabel_dict = {}
-            for kk in range(len(top_indices)):
-                if self.dl_tr.dataset.ys[top_nn_indices[kk]] != self.dl_tr.dataset.ys[top_nn_samecls_indices[kk]]: # inconsistent label between global NN and same class NN
-                    relabel_dict[top_indices[kk]] = [self.dl_tr.dataset.ys[top_nn_samecls_indices[kk]],
-                                                     self.dl_tr.dataset.ys[top_nn_indices[kk]]]
-            with open('./{}/Allrelabeldict_{}_{}.pkl'.format(base_dir, pair_ind1, pair_ind2), 'wb') as handle:
-                pickle.dump(relabel_dict, handle)
+        relabel_dict = {}
+        unique_labels, unique_counts = torch.unique(self.train_label, return_counts=True)
+        median_shots_percls = unique_counts.median().item()
+        _, prob_relabel = kNN_label_pred(query_indices=top_indices, embeddings=self.train_embedding, labels=self.train_label,
+                                         nb_classes=self.dl_tr.dataset.nb_classes(), knn_k=median_shots_percls)
 
-        elif relabel_method == 'soft_knn': # relabel by weighted kNN
-            relabel_dict = {}
-            unique_labels, unique_counts = torch.unique(self.train_label, return_counts=True)
-            median_shots_percls = unique_counts.median().item()
-            _, prob_relabel = kNN_label_pred(query_indices=top_indices, embeddings=self.train_embedding, labels=self.train_label,
-                                             nb_classes=self.dl_tr.dataset.nb_classes(), knn_k=median_shots_percls)
+        for kk in range(len(top_indices)):
+            relabel_dict[top_indices[kk]] = prob_relabel[kk].detach().cpu().numpy()
 
-            for kk in range(len(top_indices)):
-                relabel_dict[top_indices[kk]] = prob_relabel[kk].detach().cpu().numpy()
+        with open('./{}/Allrelabeldict_{}_{}_soft_knn.pkl'.format(base_dir, pair_ind1, pair_ind2), 'wb') as handle:
+            pickle.dump(relabel_dict, handle)
 
-            with open('./{}/Allrelabeldict_{}_{}_soft_knn.pkl'.format(base_dir, pair_ind1, pair_ind2), 'wb') as handle:
-                pickle.dump(relabel_dict, handle)
-
-
-        else:
-            raise NotImplemented
 
 
 if __name__ == '__main__':
@@ -199,7 +127,7 @@ def calc_relabel_dict(self, lookat_harmful, relabel_method,
         pair_ind1, pair_ind2 = int(pair_ind1), int(pair_ind2)
         if not os.path.exists('./{}/All_influence_{}_{}.npy'.format(base_dir, pair_ind1, pair_ind2)):
             # sanity check: # IS.viz_2sample(IS.dl_ev, pair_ind1, pair_ind2)
-            training_sample_by_influence, influence_values = IS.MC_estimate_forpairs(all_features, [pair_ind1], [pair_ind2])
+            training_sample_by_influence, influence_values = IS.MC_estimate_forpair(all_features, [pair_ind1], [pair_ind2])
             helpful_indices = np.where(influence_values < 0)[0]
             harmful_indices = np.where(influence_values > 0)[0]
             np.save('./{}/Allhelpful_indices_{}_{}'.format(base_dir, pair_ind1, pair_ind2), helpful_indices)
@@ -216,9 +144,8 @@ def calc_relabel_dict(self, lookat_harmful, relabel_method,
         assert len(IS.train_label) == len(train_nn_indices)
 
         '''Step 3: Save harmful indices as well as its neighboring indices'''
-        IS.calc_relabel_dict(lookat_harmful=lookat_harmful, relabel_method=relabel_method,
+        IS.calc_relabel_dict(lookat_harmful=lookat_harmful,
                              harmful_indices=harmful_indices, helpful_indices=helpful_indices,
-                             train_nn_indices=train_nn_indices, train_nn_indices_same_cls=train_nn_indices_same_cls,
                              base_dir=base_dir, pair_ind1=pair_ind1, pair_ind2=pair_ind2)
     exit()
 

Influence_function/sample_relabel.py

@@ -39,7 +39,7 @@
             print('skip')
             continue
         # sanity check: IS.viz_2sample(IS.dl_ev, wrong_ind, confuse_ind)
-        mean_deltaL_deltaD = IS.MC_estimate_forpairs([wrong_ind, confuse_ind], num_thetas=1, steps=50)
+        mean_deltaL_deltaD = IS.MC_estimate_forpair([wrong_ind, confuse_ind], num_thetas=1, steps=50)
 
         influence_values = np.asarray(mean_deltaL_deltaD)
         training_sample_by_influence = influence_values.argsort()  # ascending

evaluation/neighborhood.py

@@ -20,18 +20,18 @@
     # loss_type = 'SoftTriple_noisy_{}'.format(noisy_level); dataset_name = 'cars_noisy';  config_name = 'cars'; seed = 4
     loss_type = 'SoftTriple_noisy_{}'.format(noisy_level); dataset_name = 'inshop_noisy';  config_name = 'inshop'; seed = 3
 
-    '''============ Our Influence function =================='''
+    '''============================================= Our Empirical Influence function =============================================================='''
     IS = MCScalableIF(dataset_name, seed, loss_type, config_name, test_crop)
     basedir = 'MislabelExp_Influential_data'
     os.makedirs(basedir, exist_ok=True)
 
     for num_thetas in [1, 2, 3]:
 
         '''Mislabelled data detection'''
-        if os.path.exists("{}/{}_{}_helpful_testcls{}_SIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level)):
-            helpful_indices = np.load("{}/{}_{}_helpful_testcls{}_SIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
-            harmful_indices = np.load("{}/{}_{}_harmful_testcls{}_SIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
-            influence_values = np.load("{}/{}_{}_influence_values_testcls{}_SIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
+        if os.path.exists("{}/{}_{}_helpful_testcls{}_EIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level)):
+            helpful_indices = np.load("{}/{}_{}_helpful_testcls{}_EIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
+            harmful_indices = np.load("{}/{}_{}_harmful_testcls{}_EIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
+            influence_values = np.load("{}/{}_{}_influence_values_testcls{}_EIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level))
         else:
             confusion_class_pairs = IS.get_confusion_class_pairs()
 
@@ -46,9 +46,9 @@
 
             helpful_indices = np.where(influence_values < 0)[0]  # cache all helpful
             harmful_indices = np.where(influence_values > 0)[0]  # cache all harmful
-            np.save("{}/{}_{}_helpful_testcls{}_SIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), helpful_indices)
-            np.save("{}/{}_{}_harmful_testcls{}_SIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), harmful_indices)
-            np.save("{}/{}_{}_influence_values_testcls{}_SIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), influence_values)
+            np.save("{}/{}_{}_helpful_testcls{}_EIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), helpful_indices)
+            np.save("{}/{}_{}_harmful_testcls{}_EIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), harmful_indices)
+            np.save("{}/{}_{}_influence_values_testcls{}_EIF_theta{}_{}".format(basedir, IS.dataset_name, IS.loss_type, 0, num_thetas, noisy_level), influence_values)
 
         training_sample_by_influence = influence_values.argsort()  # ascending, harmful first
         # mislabelled indices ground-truth
@@ -68,7 +68,7 @@
     # TODO climbing plot
     '''Weighted KNN'''
     start_time = time.time()
-    harmful_indices = np.load("{}/{}_{}_harmful_testcls{}_SIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, 1, noisy_level))
+    harmful_indices = np.load("{}/{}_{}_harmful_testcls{}_EIF_theta{}_{}.npy".format(basedir, IS.dataset_name, IS.loss_type, 0, 1, noisy_level))
     relabel_dict = {}
     unique_labels, unique_counts = torch.unique(IS.train_label, return_counts=True)
     median_shots_percls = unique_counts.median().item()
@@ -92,8 +92,9 @@
                     ct_correct += 1
     print(ct_correct, total_ct)
 
+    '''======================================================================================================================================='''
 
-    '''============ Original Influence function =================='''
+    '''======================================== Original Influence function =========================================================================='''
     IS = OrigIF(dataset_name, seed, loss_type, config_name, test_crop)
     basedir = 'MislabelExp_Influential_data'
     os.makedirs(basedir, exist_ok=True)
@@ -138,7 +139,9 @@
 
     plt.plot(cum_overlap, label='IF')
 
-    '''Relabelled data accuracy (only relabel harmful)'''
+    '''======================================================================================================================================='''
+
+    '''=============================================Random================================================================'''
     overlap = np.isin(np.arange(len(IS.dl_tr.dataset)), gt_mislabelled_indices)
     cum_overlap = np.cumsum(overlap)
 
@@ -147,6 +150,7 @@
     plt.tight_layout()
     plt.savefig('./images/mislabel_{}_{}_alltheta_noisylevel{}.pdf'.format(dataset_name, loss_type, noisy_level),
                 bbox_inches='tight')
-    # plt.savefig('./images/mislabel_{}_{}_alltheta_noisylevel{}.png'.format(dataset_name, loss_type, noisy_level),
-    #             bbox_inches='tight')
+
+    '''======================================================================================================================================='''
+
 

experiments/EIF_pair_confusion.py

@@ -46,11 +46,9 @@
                 helpful_indices)
         np.save("Influential_data_baselines/{}_{}_harmful_testcls{}".format(IS.dataset_name, IS.loss_type, pair_idx),
                 harmful_indices)
-    exit()
 
     '''Actually train with downweighted harmful and upweighted helpful training'''
     os.system("./scripts/run_{}_IF_{}.sh".format(dataset_name, loss_type))
-    exit()
 
     '''Other: get confusion (before VS after)'''
     IS.model = IS._load_model()  # reload the original weights
@@ -75,5 +73,3 @@
         inter_dist_after, _ = grad_confusion(IS.model, features, wrong_cls, confuse_classes,
                                              IS.testing_nn_label, IS.testing_label, IS.testing_nn_indices)
         print("After d(G_p): ", inter_dist_after)
-
-    exit()