Update finetune.py

finetune.py

@@ -17,12 +17,12 @@ from peft import (
 
 
 # Parameters
-MICRO_BATCH_SIZE = int(sys.argv[2])
+MICRO_BATCH_SIZE = int(32)
 BATCH_SIZE = 16
-size = sys.argv[1]
+size = 7b
 GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
 EPOCHS = 1
-LEARNING_RATE = float(sys.argv[3])
+LEARNING_RATE = float(0.00015)
 CUTOFF_LEN = 512
 LORA_R = 8
 LORA_ALPHA = 16
@@ -43,7 +43,7 @@ if not os.path.exists("data"):
     os.makedirs("data")
 # Load data
 data = []
-for x in sys.argv[4].split(","):
+for x in "alpaca,stackoverflow,quora".split(","):
     data += json.load(open("data/{}_chat_data.json".format(x)))
 random.shuffle(data)
 json.dump(data, open(DATA_PATH, "w"))
@@ -161,9 +161,6 @@ model.state_dict = (
     lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict())
 ).__get__(model, type(model))
 
-if torch.__version__ >= "2" and sys.platform != "win32":
-    model = torch.compile(model)
-
 trainer.train()
 
 model.save_pretrained(OUTPUT_DIR)