changed source code

1 #!/usr/bin/env rdmd
2 
3 // Written in the D programming language
4 
5 /**
6 Change log generator which fetches the list of bugfixes
7 from the D Bugzilla between the given dates.
8 Moreover manual changes are accumulated from raw text files in the
9 Dlang repositories.
10 It stores its result in DDoc form to a text file.
11 
12 Copyright: D Language Foundation 2016.
13 
14 License:   $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0).
15 
16 Authors:   Dmitry Olshansky,
17            Andrej Mitrovic,
18            Sebastian Wilzbach
19 
20 Example usage:
21 
22 ---
23 rdmd changed.d "v2.071.2..upstream/stable"
24 ---
25 
26 It is also possible to directly preview the generated changelog file:
27 
28 ---
29 rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/web/changelog/pending.html
30 ---
31 
32 If no arguments are passed, only the manual changes will be accumulated and Bugzilla
33 won't be queried (faster).
34 
35 A manual changelog entry consists of a title line, a blank separator line and
36 the description.
37 */
38 
39 // NOTE: this script requires libcurl to be linked in (usually done by default).
40 
41 module changed;
42 
43 import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typecons,
44     std.stdio, std.datetime, std.array, std..string, std.file, std.format, std.getopt,
45     std.path;
46 
47 import std.range.primitives, std.traits;
48 
49 struct BugzillaEntry
50 {
51     int id;
52     string summary;
53 }
54 
55 struct ChangelogEntry
56 {
57     string title; // the first line (can't contain links)
58     string description; // a detailed description (separated by a new line)
59     string basename; // basename without extension (used for the anchor link to the description)
60     string repo; // origin repository that contains the changelog entry
61     string filePath; // path to the changelog entry (relative from the repository root)
62 }
63 
64 // Also retrieve new (but not reopened) bugs, as bugs are only auto-closed when
65 // merged into master, but the changelog gets generated on stable.
66 auto templateRequest =
67     `https://issues.dlang.org/buglist.cgi?bug_id={buglist}&bug_status=NEW&bug_status=RESOLVED&`~
68         `ctype=csv&columnlist=component,bug_severity,short_desc`;
69 
70 auto generateRequest(Range)(string templ, Range issues)
71 {
72     auto buglist = format("%(%d,%)", issues);
73     return templateRequest.replace("{buglist}", buglist);
74 }
75 
76 auto dateFromStr(string sdate)
77 {
78     int year, month, day;
79     formattedRead(sdate, "%s-%s-%s", &year, &month, &day);
80     return Date(year, month, day);
81 }
82 
83 string[dchar] parenToMacro;
84 shared static this()
85 {
86     parenToMacro = ['(' : "$(LPAREN)", ')' : "$(RPAREN)"];
87 }
88 
89 /** Replace '(' and ')' with macros to avoid closing down macros by accident. */
90 string escapeParens(string input)
91 {
92     return input.translate(parenToMacro);
93 }
94 
95 /** Get a list of all bugzilla issues mentioned in revRange */
96 auto getIssues(string revRange)
97 {
98     import std.process : execute, pipeProcess, Redirect, wait;
99     import std.regex : ctRegex, match, splitter;
100 
101     // see https://github.com/github/github-services/blob/2e886f407696261bd5adfc99b16d36d5e7b50241/lib/services/bugzilla.rb#L155
102     enum closedRE = ctRegex!(`((close|fix|address)e?(s|d)? )?(ticket|bug|tracker item|issue)s?:? *([\d ,\+&#and]+)`, "i");
103 
104     auto issues = appender!(int[]);
105     foreach (repo; ["dmd", "druntime", "phobos", "dlang.org", "tools", "installer"]
106              .map!(r => buildPath("..", r)))
107     {
108         auto cmd = ["git", "-C", repo, "fetch", "--tags", "https://github.com/dlang/" ~ repo.baseName,
109                            "+refs/heads/*:refs/remotes/upstream/*"];
110         auto p = pipeProcess(cmd, Redirect.stdout);
111         enforce(wait(p.pid) == 0, "Failed to execute '%(%s %)'.".format(cmd));
112 
113         cmd = ["git", "-C", repo, "log", revRange];
114         p = pipeProcess(cmd, Redirect.stdout);
115         scope(exit) enforce(wait(p.pid) == 0, "Failed to execute '%(%s %)'.".format(cmd));
116 
117         foreach (line; p.stdout.byLine())
118         {
119             if (auto m = match(line, closedRE))
120             {
121                 if (!m.captures[1].length) continue;
122                 m.captures[5]
123                     .splitter(ctRegex!`[^\d]+`)
124                     .filter!(b => b.length)
125                     .map!(to!int)
126                     .copy(issues);
127             }
128         }
129     }
130     return issues.data.sort().release.uniq;
131 }
132 
133 /** Generate and return the change log as a string. */
134 auto getBugzillaChanges(string revRange)
135 {
136     // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries
137     BugzillaEntry[][string][string] entries;
138 
139     auto issues = getIssues(revRange);
140     // abort prematurely if no issues are found in all git logs
141     if (issues.empty)
142         return entries;
143 
144     auto req = generateRequest(templateRequest, issues);
145     debug stderr.writeln(req);  // write text
146     auto data = req.get;
147 
148     foreach (fields; csvReader!(Tuple!(int, string, string, string))(data, null))
149     {
150         string comp = fields[1].toLower;
151         switch (comp)
152         {
153             case "dlang.org": comp = "dlang.org"; break;
154             case "dmd": comp = "DMD Compiler"; break;
155             case "druntime": comp = "Druntime"; break;
156             case "installer": comp = "Installer"; break;
157             case "phobos": comp = "Phobos"; break;
158             case "tools": comp = "Tools"; break;
159             case "dub": comp = "Dub"; break;
160             case "visuald": comp = "VisualD"; break;
161             default: assert(0, comp);
162         }
163 
164         string type = fields[2].toLower;
165         switch (type)
166         {
167             case "regression":
168                 type = "regressions";
169                 break;
170 
171             case "blocker", "critical", "major", "normal", "minor", "trivial":
172                 type = "bugs";
173                 break;
174 
175             case "enhancement":
176                 type = "enhancements";
177                 break;
178 
179             default: assert(0, type);
180         }
181 
182         entries[comp][type] ~= BugzillaEntry(fields[0], fields[3].idup);
183     }
184     return entries;
185 }
186 
187 /**
188 Reads a single changelog file.
189 
190 An entry consists of a title line, a blank separator line and
191 the description
192 
193 Params:
194     filename = changelog file to be parsed
195     repoName = origin repository that contains the changelog entry
196 
197 Returns: The parsed `ChangelogEntry`
198 */
199 ChangelogEntry readChangelog(string filename, string repoName)
200 {
201     import std.algorithm.searching : countUntil;
202     import std.file : read;
203     import std.path : baseName, stripExtension;
204     import std..string : strip;
205 
206     auto lines = filename.readText().splitLines();
207 
208     // filter empty files
209     if (lines.empty)
210         return ChangelogEntry.init;
211 
212     // filter ddoc files
213     if (lines[0].startsWith("Ddoc"))
214         return ChangelogEntry.init;
215 
216     enforce(lines.length >= 3 &&
217         !lines[0].empty &&
218          lines[1].empty &&
219         !lines[2].empty,
220         "Changelog entries should consist of one title line, a blank separator line, and a description.");
221 
222     ChangelogEntry entry = {
223         title: lines[0].strip,
224         description: lines[2..$].join("\n").strip,
225         basename: filename.baseName.stripExtension,
226         repo: repoName,
227         filePath: filename.findSplitAfter(repoName)[1].findSplitAfter("/")[1],
228     };
229     return entry;
230 }
231 
232 /**
233 Looks for changelog files (ending with `.dd`) in a directory and parses them.
234 
235 Params:
236     changelogDir = directory to search for changelog files
237     repoName = origin repository that contains the changelog entry
238 
239 Returns: An InputRange of `ChangelogEntry`s
240 */
241 auto readTextChanges(string changelogDir, string repoName)
242 {
243     import std.algorithm.iteration : filter, map;
244     import std.file : dirEntries, SpanMode;
245     import std..string : endsWith;
246 
247     return dirEntries(changelogDir, SpanMode.shallow)
248             .filter!(a => a.name().endsWith(".dd"))
249             .array.sort()
250             .map!(a => readChangelog(a, repoName))
251             .filter!(a => a.title.length > 0);
252 }
253 
254 /**
255 Writes the overview headline of the manually listed changes in the ddoc format as list.
256 
257 Params:
258     changes = parsed InputRange of changelog information
259     w = Output range to use
260 */
261 void writeTextChangesHeader(Entries, Writer)(Entries changes, Writer w, string headline)
262     if (isInputRange!Entries && isOutputRange!(Writer, string))
263 {
264     // write the overview titles
265     w.formattedWrite("$(BUGSTITLE_TEXT_HEADER %s,\n\n", headline);
266     scope(exit) w.put("\n)\n\n");
267     foreach(change; changes)
268     {
269         w.formattedWrite("$(LI $(RELATIVE_LINK2 %s,%s))\n", change.basename, change.title);
270     }
271 }
272 /**
273 Writes the long description of the manually listed changes in the ddoc format as list.
274 
275 Params:
276     changes = parsed InputRange of changelog information
277     w = Output range to use
278 */
279 void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string headline)
280     if (isInputRange!Entries && isOutputRange!(Writer, string))
281 {
282     w.formattedWrite("$(BUGSTITLE_TEXT_BODY %s,\n\n", headline);
283     scope(exit) w.put("\n)\n\n");
284     foreach(change; changes)
285     {
286         w.formattedWrite("$(LI $(LNAME2 %s,%s)\n", change.basename, change.title);
287         w.formattedWrite("$(CHANGELOG_SOURCE_FILE %s, %s)\n", change.repo, change.filePath);
288         scope(exit) w.put(")\n\n");
289 
290         bool inPara, inCode;
291         foreach (line; change.description.splitLines)
292         {
293             if (line.stripLeft.startsWith("---"))
294             {
295                 if (inPara)
296                 {
297                     w.put(")\n");
298                     inPara = false;
299                 }
300                 inCode = !inCode;
301             }
302             else if (!inCode && !inPara && !line.empty)
303             {
304                 w.put("$(P\n");
305                 inPara = true;
306             }
307             else if (inPara && line.empty)
308             {
309                 w.put(")\n");
310                 inPara = false;
311             }
312             w.put(line);
313             w.put("\n");
314         }
315         if (inPara)
316             w.put(")\n");
317     }
318 }
319 
320 /**
321 Writes the fixed issued from Bugzilla in the ddoc format as a single list.
322 
323 Params:
324     changes = parsed InputRange of changelog information
325     w = Output range to use
326 */
327 void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w)
328     if (isOutputRange!(Writer, string))
329 {
330     immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"];
331     immutable bugtypes = ["regressions", "bugs", "enhancements"];
332 
333     foreach (component; components)
334     {
335         if (auto comp = component in entries)
336         {
337             foreach (bugtype; bugtypes)
338             if (auto bugs = bugtype in *comp)
339             {
340                 w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype);
341                 foreach (bug; sort!"a.id < b.id"(*bugs))
342                 {
343                     w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n",
344                                         bug.id, bug.summary.escapeParens());
345                 }
346                 w.put(")\n");
347             }
348         }
349     }
350 }
351 
352 int main(string[] args)
353 {
354     auto outputFile = "./changelog.dd";
355     auto nextVersionString = "LATEST";
356 
357     auto currDate = Clock.currTime();
358     auto nextVersionDate = "%s %02d, %04d"
359         .format(currDate.month.to!string.capitalize, currDate.day, currDate.year);
360 
361     string previousVersion = "Previous version";
362     bool hideTextChanges = false;
363     string revRange;
364 
365     auto helpInformation = getopt(
366         args,
367         std.getopt.config.passThrough,
368         "output|o", &outputFile,
369         "date", &nextVersionDate,
370         "version", &nextVersionString,
371         "prev-version", &previousVersion, // this can automatically be detected
372         "no-text", &hideTextChanges);
373 
374     if (helpInformation.helpWanted)
375     {
376 `Changelog generator
377 Please supply a bugzilla version
378 ./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options);
379     }
380 
381     if (args.length >= 2)
382     {
383         revRange = args[1];
384 
385         // extract the previous version
386         auto parts = revRange.split("..");
387         if (parts.length > 1)
388             previousVersion = parts[0].replace("v", "");
389     }
390     else
391     {
392         writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable");
393     }
394 
395     // location of the changelog files
396     alias Repo = Tuple!(string, "name", string, "headline", string, "path");
397     auto repos = [Repo("dmd", "Compiler changes", null),
398                   Repo("druntime", "Runtime changes", null),
399                   Repo("phobos", "Library changes", null),
400                   Repo("dlang.org", "Language changes", null),
401                   Repo("installer", "Installer changes", null),
402                   Repo("tools", "Tools changes", null),
403                   Repo("dub", "Dub changes", null)];
404 
405     auto changedRepos = repos
406          .map!(repo => Repo(repo.name, repo.headline, buildPath(__FILE_FULL_PATH__.dirName, "..", repo.name, repo.name == "dlang.org" ? "language-changelog" : "changelog")))
407          .filter!(r => r.path.exists);
408 
409     // ensure that all files either end on .dd or .md
410     bool errors;
411     foreach (repo; changedRepos)
412     {
413         auto invalidFiles = repo.path
414             .dirEntries(SpanMode.shallow)
415             .filter!(a => !a.name.endsWith(".dd", ".md"));
416         if (!invalidFiles.empty)
417         {
418             invalidFiles.each!(f => stderr.writefln("ERROR: %s needs to have .dd or .md as extension", f.buildNormalizedPath));
419             errors = 1;
420         }
421     }
422     import core.stdc.stdlib : exit;
423     if (errors)
424         1.exit;
425 
426     auto f = File(outputFile, "w");
427     auto w = f.lockingTextWriter();
428     w.put("Ddoc\n\n");
429     w.put("$(CHANGELOG_NAV_INJECT)\n\n");
430 
431     {
432         w.formattedWrite("$(VERSION %s, =================================================,\n\n", nextVersionDate);
433 
434         scope(exit) w.put(")\n");
435 
436         if (!hideTextChanges)
437         {
438             // search for raw change files
439             auto changelogDirs = changedRepos
440                  .map!(r => tuple!("headline", "changes")(r.headline, r.path.readTextChanges(r.name).array))
441                  .filter!(r => !r.changes.empty);
442 
443             // print the overview headers
444             changelogDirs.each!(r => r.changes.writeTextChangesHeader(w, r.headline));
445 
446             if (!revRange.empty)
447                 w.put("$(CHANGELOG_SEP_HEADER_TEXT_NONEMPTY)\n\n");
448 
449             w.put("$(CHANGELOG_SEP_HEADER_TEXT)\n\n");
450 
451             // print the detailed descriptions
452             changelogDirs.each!(x => x.changes.writeTextChangesBody(w, x.headline));
453 
454             if (revRange.length)
455                 w.put("$(CHANGELOG_SEP_TEXT_BUGZILLA)\n\n");
456         }
457         else
458         {
459                 w.put("$(CHANGELOG_SEP_NO_TEXT_BUGZILLA)\n\n");
460         }
461 
462         // print the entire changelog history
463         if (revRange.length)
464             revRange.getBugzillaChanges.writeBugzillaChanges(w);
465     }
466 
467     version(Contributors_Lib)
468     if (revRange)
469     {
470         import contributors : FindConfig, findAuthors, reduceAuthors;
471         FindConfig config = {
472             cwd: __FILE_FULL_PATH__.dirName.asNormalizedPath.to!string,
473         };
474         config.mailmapFile = config.cwd.buildPath(".mailmap");
475         auto authors = revRange.findAuthors(config).reduceAuthors;
476         w.formattedWrite("$(D_CONTRIBUTORS_HEADER %d)\n", authors.save.walkLength);
477         w.put("$(D_CONTRIBUTORS\n");
478         authors.each!(a => w.formattedWrite("    $(D_CONTRIBUTOR %s)\n", a.name));
479         w.put(")\n");
480         w.put("$(D_CONTRIBUTORS_FOOTER)\n");
481     }
482 
483     w.put("$(CHANGELOG_NAV_INJECT)\n\n");
484 
485     // write own macros
486     w.formattedWrite(`Macros:
487     VER=%s
488     TITLE=Change Log: $(VER)
489 `, nextVersionString);
490 
491     writefln("Change log generated to: '%s'", outputFile);
492     return 0;
493 }