init

app.py

@@ -39,8 +39,13 @@ def main():
     st.write(f"Prompt Type: {prompt_type}")  
   
     if st.button("Retrieve"):  
-        output = retrieve(model_name, dataset_name, attack_name, prompt_type)  
-        st.write(f"Output: {output}")  
+        results = retrieve(model_name, dataset_name, attack_name, prompt_type)
+        
+        for result in results:
+            st.write("Original prompt: {}".format(result["origin prompt"]))
+            st.write("Original acc: {}".format(result["origin acc"]))
+            st.write("Attack prompt: {}".format(result["attack prompt"]))
+            st.write("Attack acc: {}".format(result["attack acc"]))
   
 if __name__ == "__main__":  
     main()  

parse.py

@@ -120,13 +120,32 @@ def retrieve(model_name, dataset_name, attack_name, prompt_type):
     else:
         prompt_type = "role"
     
-    directory_path = "./db"
+    directory_path = "./adv_prompts"
     md_dir = os.path.join(directory_path, model_name + "_" + shot + ".md")
     sections_dict = split_markdown_by_title(md_dir)
     results = {}
     for cur_dataset in sections_dict.keys():
         if cur_dataset == dataset_name:
             dataset_dict = sections_dict[cur_dataset]
+            best_acc = 0
+            best_prompt = ""
+            for cur_attack in dataset_dict.keys():
+                if cur_attack == "10 prompts":
+                    prompts_dict = dataset_dict[cur_attack].split("\n")
+                    num = 0
+                    for prompt_summary in prompts_dict:
+                        if "Acc: " not in prompt_summary:
+                            continue
+                        else:
+                            import re
+                            num += 1
+                            match = re.search(r'Acc: (\d+\.\d+)%', prompt_summary)
+                            if match:
+                                number = float(match.group(1))
+                                if number > best_acc:
+                                    best_acc = number
+                                    best_prompt = prompt_summary.split("prompt: ")[1]
+
             for cur_attack in dataset_dict.keys():
                 
                 if cur_attack == attack_name:
@@ -142,63 +161,52 @@ def retrieve(model_name, dataset_name, attack_name, prompt_type):
                             
                             import re
                         
-                            match_atk = re.search(r'acc: (\d+\.\d+)%', result)
+                            match_atk = re.search(r'acc: (\d+\.\d+)%', prompt_summary)
                             number_atk = float(match_atk.group(1))
                             results[prompt] = number_atk
   
                         sorted_results = sorted(results.items(), key=lambda item: item[1])[:6]
-      
 
-                        return sorted_results
+                        returned_results = []
+                        for result in sorted_results:
+                            returned_results.append({"origin prompt": best_prompt, "origin acc": best_acc, "attack prompt": result[0], "attack acc": result[1]})
+      
+                        return returned_results
 
                     elif attack_name in ["bertattack", "checklist", "deepwordbug", "stresstest", "textfooler", "textbugger"]:
 
-                        prompts_dict = dataset_dict[attack_name].split("\n")
+                        prompts_dict = dataset_dict[attack_name].split("Original prompt: ")
                         num = 0
                         
-
+                        returned_results = []
                         for prompt_summary in prompts_dict:
                             if "Attacked prompt: " not in prompt_summary:
                                 continue
 
+                            origin_prompt = prompt_summary.split("\n")[0]
+                            attack_prompt = prompt_summary.split("Attacked prompt: ")[1].split("Original acc: ")[0]
+                            attack_prompt = bytes(attack_prompt[2:-1], "utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")
+
+                            print(origin_prompt)
+                            print(attack_prompt)
+
                             num += 1
                             import re
                             match_origin = re.search(r'Original acc: (\d+\.\d+)%', prompt_summary)
                             match_atk = re.search(r'attacked acc: (\d+\.\d+)%', prompt_summary)
-                        if match_origin and match_atk:
-                            number_origin = float(match_origin.group(1))
-                            number_atk = float(match_atk.group(1))
-                    
-                    # print(model_shot, dataset, title, len(summary[attack][dataset]), num)
-    
-    # for atk in summary.keys():
-    #     for dataset in summary[atk].keys():
-    #         # if atk == "translation":
-    #         print(atk, dataset, len(summary[atk][dataset]))
-    #             # print(summary[atk][dataset][:10])
-    
-    output_dict = {}
+                            if match_origin and match_atk:
+                                if prompt_type == "task":
+                                    if num > 3:
+                                        break
+                                else:
+                                    if num < 3:
+                                        continue
+                                number_origin = float(match_origin.group(1))
+                                number_atk = float(match_atk.group(1))
+                                returned_results.append({"origin prompt": origin_prompt, "origin acc": number_origin, "attack prompt": attack_prompt, "attack acc": number_atk})
+                        
+                        return returned_results
 
-    sorted_atk_name = ["TextBugger",  "DeepWordBug", "TextFooler", "BertAttack", "CheckList", "StressTest", "Semantic"]
-    sorted_dataset_name = ["SST-2", "CoLA", "QQP", "MRPC", "MNLI", "QNLI", "RTE",  "WNLI", "MMLU", "SQuAD V2", "IWSLT", "UN Multi", "Math"]
-    
-    for atk in sorted_atk_name:
-        output_dict[atk] = {}
-        for dataset in sorted_dataset_name:
-            output_dict[atk][dataset] = ""
-    
-    for sorted_atk in sorted_atk_name:
-        for attack, dataset_drop_rates in summary.items():
-            # attack = convert_attack_name(attack)
-            if convert_attack_name(attack) == sorted_atk:
-                for sorted_dataset in sorted_dataset_name:
-                    for dataset, drop_rates in dataset_drop_rates.items():
-                        if convert_dataset_name(dataset) == sorted_dataset:
-                            if len(drop_rates) > 0:
-                                output_dict[sorted_atk][sorted_dataset] = "{:.2f}".format(sum(drop_rates)/len(drop_rates)) + "\scriptsize{$\pm$" + "{:.2f}".format(np.std(drop_rates)) + "}"
-                            else:
-                                output_dict[sorted_atk][sorted_dataset] = "-"
-
-                total_drop_rate = summary[attack]["Avg"]
-                output_dict[sorted_atk]["Avg"] = "{:.2f}".format(np.mean(total_drop_rate)) + "\scriptsize{$\pm$" + "{:.2f}".format(np.std(total_drop_rate)) + "}"
-    
+
+if __name__ == "__main__":
+    print(retrieve("T5", "CoLA", "BertAttack", "zeroshot_task"))