add prompt param

Author: tmori

critical_thinking.py

@@ -35,11 +35,14 @@ def save_to_json(self, file_path):
 
 if __name__ == "__main__":
     import sys
+    from params import get_param
+    prompt_template_path = get_param("prompt_templates_path")
+
     if len(sys.argv) != 3:
         print("Usage: <MainQuestion> <BackgroundKnowledge>")
         sys.exit(1)
     main_question = sys.argv[1]
     background_knowledge_path = sys.argv[2]
-    think = CriticalThinking(main_question, "./prompt_templates/ptemplate_critical_thinking.txt", background_knowledge_path)
+    think = CriticalThinking(main_question, prompt_template_path + "/ptemplate_critical_thinking.txt", background_knowledge_path)
     think.create()
     think.save_to_raw("test/result/critical_thinking.json")

evaluator.py

@@ -69,6 +69,8 @@ def evaluate(self, template_path, ref_json_path):
 
 if __name__ == "__main__":
     import sys
+    from params import get_param
+    prompt_template_path = get_param("prompt_templates_path")
 
     if len(sys.argv) != 4 and len(sys.argv) != 5:
         print("Usage: <MainQuestion> <plan> <memory> [<reflection>]")
@@ -78,12 +80,12 @@ def evaluate(self, template_path, ref_json_path):
     mem_json_path = sys.argv[3]
     plan = Plan()
     plan.load_from_json(plan_json_path)
-    mission_path= "./prompt_templates/ptemplate_mission.txt"
+    mission_path= prompt_template_path + "/ptemplate_mission.txt"
     memory_stream = MemoryStream()
     memory_stream.load_from_json(mem_json_path)
     evaluator = Evaluator(main_question, mission_path, plan, memory_stream)
     if len(sys.argv) == 4:
         evaluator.merge_data()
     elif len(sys.argv) == 5:
         ref_json_path = sys.argv[4]
-        evaluator.evaluate("./prompt_templates/ptemplate_evaluate.txt", ref_json_path)
+        evaluator.evaluate(prompt_template_path + "/ptemplate_evaluate.txt", ref_json_path)

params.json

@@ -0,0 +1,6 @@
+{
+    "prompt_templates_path": "./prompt_templates",
+    "documents_path": "../documents",
+    "documents_data_path": "./documents_data",
+    "reflections_data_path": "./reflections_data"
+}

params.py

@@ -0,0 +1,18 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import json
+
+def get_param(param_name: str):
+    with open('./params.json', 'r') as file:
+        param = json.load(file)
+        return param.get(param_name)
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        print("Usage: <param_name>")
+        sys.exit(1)
+    param_name = sys.argv[1]
+
+    print(get_param(param_name))

planner.py

@@ -111,6 +111,8 @@ def save_strategy_history(self):
 
 if __name__ == "__main__":
     import sys
+    from params import get_param
+    prompt_template_path = get_param("prompt_templates_path")
     if len(sys.argv) != 5:
         print("Usage: <MainQuestion> <doc_list.txt> <background_knowledge_path> <acquired_knowledge_path>")
         sys.exit(1)
@@ -124,9 +126,9 @@ def save_strategy_history(self):
         batched_list = [total_list[i:i+batch_size] for i in range(0, len(total_list), batch_size)]
     planner = Planner(
         main_question = main_question,
-        mission_path= "./prompt_templates/ptemplate_mission.txt",
-        strategy_path= "./prompt_templates/ptemplate_strategy.txt",
-        query_plan_path= "./prompt_templates/ptemplate_query_plan.txt",
+        mission_path= prompt_template_path + "/ptemplate_mission.txt",
+        strategy_path= prompt_template_path + "/ptemplate_strategy.txt",
+        query_plan_path= prompt_template_path + "/ptemplate_query_plan.txt",
         strategy_history_path="./test/strategy_history.json",
         background_knowledge_path = background_knowledge_path,
         acquired_knowledge_path = acquired_knowledge_path

prompt_template.py

@@ -15,7 +15,10 @@ def get_prompt(self, **kwargs) -> str:
         return self.template.format(**kwargs)
 
 if __name__ == "__main__":
-    pt = PromptTemplate("./prompt_templates/ptemplate_query.txt")
+    from params import get_param
+    prompt_template_path = get_param("prompt_templates_path")
+
+    pt = PromptTemplate(prompt_template_path + "/ptemplate_query.txt")
     while True:
         target_doc_id = input("TargetDocID> ")
         question = input("question> ")

query.py

@@ -46,11 +46,14 @@ def _save(self, sub_question: str, reply: str, point: int):
 if __name__ == "__main__":
     import sys
     from db_manager import get_qa
+    from params import get_param
+    param_prompt_template_path = get_param("prompt_templates_path")
+    
     db_dir = ".."
     doc_id = "DB"
     qa = get_qa(db_dir, doc_id)
     memory_stream = MemoryStream()
-    prompt_template_path = "./prompt_templates/ptemplate_query.txt"
+    prompt_template_path = param_prompt_template_path + "/ptemplate_query.txt"
     query = Query("1", "Athrillとは何ですか？", memory_stream, qa)
     while True:
         question = input("question> ")

tactical_plannig.py

@@ -41,22 +41,25 @@ def _generate_document_question(self, prompt_template_path, document_id, purpose
 if __name__ == "__main__":
     from query import Query
     from memory_stream import MemoryStream
+    from params import get_param
+    param_prompt_template_path = get_param("prompt_templates_path")
+
     plan = Plan()
     plan.load_from_json("./test/result/plan.json")
     db_dir = "../documents/dbs"
     tactical_planning = TacticalPlanning(plan, db_dir)
     memory_stream = MemoryStream()
 
     while True:
-        ret = tactical_planning.generate_question("./prompt_templates/ptemplate_subq_detail.txt")
+        ret = tactical_planning.generate_question(param_prompt_template_path + "/ptemplate_subq_detail.txt")
         if ret == None:
             print("END")
             break
         plan_id = ret[0]
         doc_id = ret[1]
         question = ret[2]
         qa = get_qa(db_dir, doc_id)
-        prompt_template_path = "./prompt_templates/ptemplate_query.txt"
+        prompt_template_path = param_prompt_template_path + "/ptemplate_query.txt"
         query = Query(doc_id, question, memory_stream, qa)
         memory_id = query.run(prompt_template_path, question)
         if memory_id < 0:

tools/do_query.bash

@@ -7,6 +7,31 @@ then
     exit 1
 fi
 
+PRMT_TMP_PATH=`python3 params.py "prompt_templates_path"`
+if [ ! -d ${PRMT_TMP_PATH} ]
+then
+    echo "ERROR: can not found path: ${PRMT_TMP_PATH}"
+    exit 1
+fi
+DOCUMENT_PATH=`python3 params.py "documents_path"`
+if [ ! -d ${DOCUMENT_PATH} ]
+then
+    echo "ERROR: can not found path: ${DOCUMENT_PATH}"
+    exit 1
+fi
+DOC_DATA_PATH=`python3 params.py "documents_data_path"`
+if [ ! -d ${DOC_DATA_PATH} ]
+then
+    echo "ERROR: can not found path: ${DOC_DATA_PATH}"
+    exit 1
+fi
+REF_DATA_PATH=`python3 params.py "reflections_data_path"`
+if [ ! -d ${REF_DATA_PATH} ]
+then
+    echo "ERROR: can not found path: ${REF_DATA_PATH}"
+    exit 1
+fi
+
 if [ -d test ]
 then
     :
@@ -47,8 +72,8 @@ query="`cat ${query_dir}/query.txt`"
 
 DOCUMENT_TOKENS=2048
 REFLECTION_TOKENS=2048
-USE_BACKGROUND="FALSE"
-#USE_BACKGROUND="TRUE"
+#USE_BACKGROUND="FALSE"
+USE_BACKGROUND="TRUE"
 ADD_REFLECTION="TRUE"
 #ADD_REFLECTION="FALSE"
 export NEW_STARTEGY=
@@ -142,9 +167,9 @@ do
         echo "INFO: REFLECTING..."
         if [ -f "./test/result/reflection.json" ]
         then
-            python3 reflection.py "$query" ../documents/document.list "./test/result/reflection.json" ${background_file} "./prompt_templates/ptemplate_reflection.txt"
+            python3 reflection.py "$query" ../documents/document.list "./test/result/reflection.json" ${background_file} "${PRMT_TMP_PATH}/ptemplate_reflection.txt"
         else
-            python3 reflection.py "$query" ../documents/document.list "./test/result/critical_thinking.json" ${background_file} "./prompt_templates/ptemplate_reflection.txt"
+            python3 reflection.py "$query" ../documents/document.list "./test/result/critical_thinking.json" ${background_file} "${PRMT_TMP_PATH}/ptemplate_reflection.txt"
         fi
         cp ./test/result/reflection.json ./test/result/reflection_org.json
         python3 check_recover_json.py ./test/result/reflection.json
@@ -156,7 +181,7 @@ do
 
         echo "INFO: ADD REFLECTION TERMS..."
         #cp ./test/result/reflection.json ./test/result/prev_reflection.json
-        python3 reflection.py "$query" ../documents/document.list "./test/result/reflection.json" ${background_file} "./prompt_templates/ptemplate_reflection_addterms.txt"
+        python3 reflection.py "$query" ../documents/document.list "./test/result/reflection.json" ${background_file} "${PRMT_TMP_PATH}/ptemplate_reflection_addterms.txt"
         python3 check_recover_json.py ./test/result/reflection.json
         if [ $? -ne 0 ]
         then