Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place. Commercial Alternative to JupyterHub.

1
import argparse
2
import logging
3
import os
4
import random
5
import sys
6

7
import numpy as np
8
import torch
9
from datasets import load_from_disk, load_metric
10
from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments
11
from transformers.trainer_utils import get_last_checkpoint
12

13
if __name__ == "__main__":
14

15
    parser = argparse.ArgumentParser()
16

17
    # hyperparameters sent by the client are passed as command-line arguments to the script.
18
    parser.add_argument("--epochs", type=int, default=3)
19
    parser.add_argument("--train_batch_size", type=int, default=32)
20
    parser.add_argument("--eval_batch_size", type=int, default=64)
21
    parser.add_argument("--warmup_steps", type=int, default=500)
22
    parser.add_argument("--model_id", type=str)
23
    parser.add_argument("--learning_rate", type=str, default=5e-5)
24
    parser.add_argument("--fp16", type=bool, default=True)
25
    parser.add_argument("--output_dir", type=str)
26

27
    # Data, model, and output directories
28
    parser.add_argument("--output_data_dir", type=str, default=os.environ["SM_OUTPUT_DATA_DIR"])
29
    parser.add_argument("--n_gpus", type=str, default=os.environ["SM_NUM_GPUS"])
30
    parser.add_argument("--training_dir", type=str, default=os.environ["SM_CHANNEL_TRAIN"])
31
    parser.add_argument("--test_dir", type=str, default=os.environ["SM_CHANNEL_TEST"])
32

33
    args, _ = parser.parse_known_args()
34

35
    # Set up logging
36
    logger = logging.getLogger(__name__)
37

38
    logging.basicConfig(
39
        level=logging.getLevelName("INFO"),
40
        handlers=[logging.StreamHandler(sys.stdout)],
41
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
42
    )
43

44
    # load datasets
45
    train_dataset = load_from_disk(args.training_dir)
46
    test_dataset = load_from_disk(args.test_dir)
47

48
    logger.info(f" loaded train_dataset length is: {len(train_dataset)}")
49
    logger.info(f" loaded test_dataset length is: {len(test_dataset)}")
50

51
    metric = load_metric("accuracy")
52

53
    def compute_metrics(eval_pred):
54
        predictions, labels = eval_pred
55
        predictions = np.argmax(predictions, axis=1)
56
        return metric.compute(predictions=predictions, references=labels)
57

58
    # Prepare model labels - useful in inference API
59
    labels = train_dataset.features["labels"].names
60
    num_labels = len(labels)
61
    label2id, id2label = dict(), dict()
62
    for i, label in enumerate(labels):
63
        label2id[label] = str(i)
64
        id2label[str(i)] = label
65

66
    # download model from model hub
67
    model = AutoModelForSequenceClassification.from_pretrained(
68
        args.model_id, num_labels=num_labels, label2id=label2id, id2label=id2label
69
    )
70
    tokenizer = AutoTokenizer.from_pretrained(args.model_id)
71

72
    # define training args
73
    training_args = TrainingArguments(
74
        output_dir=args.output_dir,
75
        overwrite_output_dir=True if get_last_checkpoint(args.output_dir) is not None else False,
76
        num_train_epochs=args.epochs,
77
        per_device_train_batch_size=args.train_batch_size,
78
        per_device_eval_batch_size=args.eval_batch_size,
79
        warmup_steps=args.warmup_steps,
80
        fp16=args.fp16,
81
        evaluation_strategy="epoch",
82
        save_strategy="epoch",
83
        save_total_limit=2,
84
        logging_dir=f"{args.output_data_dir}/logs",
85
        learning_rate=float(args.learning_rate),
86
        load_best_model_at_end=True,
87
        metric_for_best_model="accuracy",
88
    )
89

90
    # create Trainer instance
91
    trainer = Trainer(
92
        model=model,
93
        args=training_args,
94
        compute_metrics=compute_metrics,
95
        train_dataset=train_dataset,
96
        eval_dataset=test_dataset,
97
        tokenizer=tokenizer,
98
    )
99

100
    # train model
101
    if get_last_checkpoint(args.output_dir) is not None:
102
        logger.info("***** continue training *****")
103
        last_checkpoint = get_last_checkpoint(args.output_dir)
104
        trainer.train(resume_from_checkpoint=last_checkpoint)
105
    else:
106
        trainer.train()
107

108
    # evaluate model
109
    eval_result = trainer.evaluate(eval_dataset=test_dataset)
110

111
    # writes eval result to file which can be accessed later in s3 ouput
112
    with open(os.path.join(args.output_data_dir, "eval_results.txt"), "w") as writer:
113
        print(f"***** Eval results *****")
114
        for key, value in sorted(eval_result.items()):
115
            writer.write(f"{key} = {value}\n")
116
            print(f"{key} = {value}\n")
117

118
    # Saves the model to s3 uses os.environ["SM_MODEL_DIR"] to make sure checkpointing works
119
    trainer.save_model(os.environ["SM_MODEL_DIR"])
120

121