1
#!/usr/bin/env bash
2

3

4
stderr() {
5
    echo "$@" 1>&2
6
}
7

8
usage() {
9
    b=$(basename "$0")
10
    echo $b: ERROR: "$@" 1>&2
11

12
    cat 1>&2 <<EOF
13

14
DESCRIPTION
15

16
    $(basename "$0") is the script to run continuous integration commands for
17
    go-toml on unix.
18

19
    Requires Go and Git to be available in the PATH. Expects to be ran from the
20
    root of go-toml's Git repository.
21

22
USAGE
23

24
    $b COMMAND [OPTIONS...]
25

26
COMMANDS
27

28
benchmark [OPTIONS...] [BRANCH]
29

30
    Run benchmarks.
31

32
    ARGUMENTS
33

34
        BRANCH Optional. Defines which Git branch to use when running
35
               benchmarks.
36

37
    OPTIONS
38

39
        -d      Compare benchmarks of HEAD with BRANCH using benchstats. In
40
                this form the BRANCH argument is required.
41

42
        -a      Compare benchmarks of HEAD against go-toml v1 and
43
                BurntSushi/toml.
44

45
        -html   When used with -a, emits the output as HTML, ready to be
46
                embedded in the README.
47

48
coverage [OPTIONS...] [BRANCH]
49

50
    Generates code coverage.
51

52
    ARGUMENTS
53

54
        BRANCH  Optional. Defines which Git branch to use when reporting
55
                coverage. Defaults to HEAD.
56

57
    OPTIONS
58

59
        -d      Compare coverage of HEAD with the one of BRANCH. In this form,
60
                the BRANCH argument is required. Exit code is non-zero when
61
                coverage percentage decreased.
62
EOF
63
    exit 1
64
}
65

66
cover() {
67
    branch="${1}"
68
    dir="$(mktemp -d)"
69

70
    stderr "Executing coverage for ${branch} at ${dir}"
71

72
    if [ "${branch}" = "HEAD" ]; then
73
	    cp -r . "${dir}/"
74
    else
75
	    git worktree add "$dir" "$branch"
76
    fi
77

78
    pushd "$dir"
79
    go test -covermode=atomic  -coverpkg=./... -coverprofile=coverage.out.tmp ./...
80
    grep -Ev '(fuzz|testsuite|tomltestgen|gotoml-test-decoder|gotoml-test-encoder)' coverage.out.tmp > coverage.out
81
    go tool cover -func=coverage.out
82
    echo "Coverage profile for ${branch}: ${dir}/coverage.out" >&2
83
    popd
84

85
    if [ "${branch}" != "HEAD" ]; then
86
	    git worktree remove --force "$dir"
87
    fi
88
}
89

90
coverage() {
91
    case "$1" in
92
	-d)
93
	    shift
94
	    target="${1?Need to provide a target branch argument}"
95

96
	    output_dir="$(mktemp -d)"
97
	    target_out="${output_dir}/target.txt"
98
	    head_out="${output_dir}/head.txt"
99
	    
100
	    cover "${target}" > "${target_out}"
101
	    cover "HEAD" > "${head_out}"
102

103
	    cat "${target_out}"
104
	    cat "${head_out}"
105

106
	    echo ""
107

108
	    target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')"
109
	    head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')"
110
	    echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%"
111

112
	    delta_pct=$(echo "$head_pct - $target_pct" | bc -l)
113
	    echo "Delta: ${delta_pct}"
114

115
	    if [[ $delta_pct = \-* ]]; then
116
		    echo "Regression!";
117

118
            target_diff="${output_dir}/target.diff.txt"
119
            head_diff="${output_dir}/head.diff.txt"
120
            cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}"
121
            cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}"
122

123
            diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}"
124
		    return 1
125
	    fi
126
	    return 0
127
	    ;;
128
    esac
129

130
    cover "${1-HEAD}"
131
}
132

133
bench() {
134
    branch="${1}"
135
    out="${2}"
136
    replace="${3}"
137
    dir="$(mktemp -d)"
138

139
    stderr "Executing benchmark for ${branch} at ${dir}"
140

141
    if [ "${branch}" = "HEAD" ]; then
142
    	cp -r . "${dir}/"
143
    else
144
	    git worktree add "$dir" "$branch"
145
    fi
146

147
    pushd "$dir"
148

149
    if [ "${replace}" != "" ]; then
150
        find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \;
151
        go get "${replace}"
152
    fi
153

154
    export GOMAXPROCS=2
155
    go test '-bench=^Benchmark(Un)?[mM]arshal' -count=10 -run=Nothing ./... | tee "${out}"
156
    popd
157

158
    if [ "${branch}" != "HEAD" ]; then
159
	    git worktree remove --force "$dir"
160
    fi
161
}
162

163
fmktemp() {
164
    if mktemp --version &> /dev/null; then
165
	# GNU
166
        mktemp --suffix=-$1
167
    else
168
	# BSD
169
	mktemp -t $1
170
    fi
171
}
172

173
benchstathtml() {
174
python3 - $1 <<'EOF'
175
import sys
176

177
lines = []
178
stop = False
179

180
with open(sys.argv[1]) as f:
181
    for line in f.readlines():
182
        line = line.strip()
183
        if line == "":
184
            stop = True
185
        if not stop:
186
            lines.append(line.split(','))
187

188
results = []
189
for line in reversed(lines[2:]):
190
    if len(line) < 8 or line[0] == "":
191
        continue
192
    v2 = float(line[1])
193
    results.append([
194
        line[0].replace("-32", ""),
195
        "%.1fx" % (float(line[3])/v2),  # v1
196
        "%.1fx" % (float(line[7])/v2),  # bs
197
    ])
198
# move geomean to the end
199
results.append(results[0])
200
del results[0]
201

202

203
def printtable(data):
204
    print("""
205
<table>
206
    <thead>
207
        <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr>
208
    </thead>
209
    <tbody>""")
210

211
    for r in data:
212
        print("        <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r))
213

214
    print("""     </tbody>
215
</table>""")
216

217

218
def match(x):
219
    return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0]
220

221
above = [x for x in results if match(x)]
222
below = [x for x in results if not match(x)]
223

224
printtable(above)
225
print("<details><summary>See more</summary>")
226
print("""<p>The table above has the results of the most common use-cases. The table below
227
contains the results of all benchmarks, including unrealistic ones. It is
228
provided for completeness.</p>""")
229
printtable(below)
230
print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>')
231
print("</details>")
232

233
EOF
234
}
235

236
benchmark() {
237
    case "$1" in
238
    -d)
239
        shift
240
     	target="${1?Need to provide a target branch argument}"
241

242
        old=`fmktemp ${target}`
243
        bench "${target}" "${old}"
244

245
        new=`fmktemp HEAD`
246
        bench HEAD "${new}"
247

248
        benchstat "${old}" "${new}"
249
        return 0
250
        ;;
251
    -a)
252
        shift
253

254
        v2stats=`fmktemp go-toml-v2`
255
        bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2"
256
        v1stats=`fmktemp go-toml-v1`
257
        bench HEAD "${v1stats}" "github.com/pelletier/go-toml"
258
        bsstats=`fmktemp bs-toml`
259
        bench HEAD "${bsstats}" "github.com/BurntSushi/toml"
260

261
        cp "${v2stats}" go-toml-v2.txt
262
        cp "${v1stats}" go-toml-v1.txt
263
        cp "${bsstats}" bs-toml.txt
264

265
        if [ "$1" = "-html" ]; then
266
            tmpcsv=`fmktemp csv`
267
            benchstat -format csv go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv
268
            benchstathtml $tmpcsv
269
        else
270
            benchstat go-toml-v2.txt go-toml-v1.txt bs-toml.txt
271
        fi
272

273
        rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt
274
        return $?
275
    esac
276

277
    bench "${1-HEAD}" `mktemp`
278
}
279

280
case "$1" in
281
    coverage) shift; coverage $@;;
282
    benchmark) shift; benchmark $@;;
283
    *) usage "bad argument $1";;
284
esac
285

286