1 #!/usr/bin/env rdmd 2 3 // Written in the D programming language 4 5 /** 6 Change log generator which fetches the list of bugfixes 7 from the D Bugzilla between the given dates. 8 Moreover manual changes are accumulated from raw text files in the 9 Dlang repositories. 10 It stores its result in DDoc form to a text file. 11 12 Copyright: D Language Foundation 2016. 13 14 License: $(WEB boost.org/LICENSE_1_0.txt, Boost License 1.0). 15 16 Authors: Dmitry Olshansky, 17 Andrej Mitrovic, 18 Sebastian Wilzbach 19 20 Example usage: 21 22 --- 23 rdmd changed.d "v2.071.2..upstream/stable" 24 --- 25 26 It is also possible to directly preview the generated changelog file: 27 28 --- 29 rdmd changed.d "v2.071.2..upstream/stable" && dmd ../dlang.org/macros.ddoc ../dlang.org/html.ddoc ../dlang.org/dlang.org.ddoc ../dlang.org/doc.ddoc ../dlang.org/changelog/changelog.ddoc changelog.dd -Df../dlang.org/web/changelog/pending.html 30 --- 31 32 If no arguments are passed, only the manual changes will be accumulated and Bugzilla 33 won't be queried (faster). 34 35 A manual changelog entry consists of a title line, a blank separator line and 36 the description. 37 */ 38 39 // NOTE: this script requires libcurl to be linked in (usually done by default). 40 41 module changed; 42 43 import std.net.curl, std.conv, std.exception, std.algorithm, std.csv, std.typecons, 44 std.stdio, std.datetime, std.array, std.string, std.file, std.format, std.getopt, 45 std.path, std.functional; 46 47 import std.range.primitives, std.traits; 48 49 struct BugzillaEntry 50 { 51 int id; 52 string summary; 53 } 54 55 struct ChangelogEntry 56 { 57 string title; // the first line (can't contain links) 58 string description; // a detailed description (separated by a new line) 59 string basename; // basename without extension (used for the anchor link to the description) 60 string repo; // origin repository that contains the changelog entry 61 string filePath; // path to the changelog entry (relative from the repository root) 62 } 63 64 struct ChangelogStats 65 { 66 size_t bugzillaIssues; // number of referenced bugzilla issues of this release 67 size_t changelogEntries; // number of changelog entries of this release 68 size_t contributors; // number of distinct contributors that have contributed to this release 69 70 /** 71 Adds a changelog entry to the summary statistics. 72 73 Params: 74 entry = changelog entry 75 */ 76 void addChangelogEntry(const ref ChangelogEntry entry) 77 { 78 changelogEntries++; 79 } 80 81 /** 82 Adds a Bugzilla issue to the summary statistics. 83 84 Params: 85 entry = bugzilla entry 86 component = component of the bugzilla issue (e.g. "dmd" or "phobos") 87 type = type of the bugzilla issue (e.g. "regression" or "blocker") 88 */ 89 void addBugzillaIssue(const ref BugzillaEntry, string component, string type) 90 { 91 bugzillaIssues++; 92 } 93 } 94 ChangelogStats changelogStats; 95 96 97 // Also retrieve new (but not reopened) bugs, as bugs are only auto-closed when 98 // merged into master, but the changelog gets generated on stable. 99 auto templateRequest = 100 `https://issues.dlang.org/buglist.cgi?bug_id={buglist}&bug_status=NEW&bug_status=RESOLVED&`~ 101 `ctype=csv&columnlist=component,bug_severity,short_desc`; 102 103 auto generateRequest(Range)(string templ, Range issues) 104 { 105 auto buglist = format("%(%d,%)", issues); 106 return templateRequest.replace("{buglist}", buglist); 107 } 108 109 auto dateFromStr(string sdate) 110 { 111 int year, month, day; 112 formattedRead(sdate, "%s-%s-%s", &year, &month, &day); 113 return Date(year, month, day); 114 } 115 116 string[dchar] parenToMacro; 117 shared static this() 118 { 119 parenToMacro = ['(' : "$(LPAREN)", ')' : "$(RPAREN)"]; 120 } 121 122 /** Replace '(' and ')' with macros to avoid closing down macros by accident. */ 123 string escapeParens(string input) 124 { 125 return input.translate(parenToMacro); 126 } 127 128 /** Get a list of all bugzilla issues mentioned in revRange */ 129 auto getIssues(string revRange) 130 { 131 import std.process : execute, pipeProcess, Redirect, wait; 132 import std.regex : ctRegex, match, splitter; 133 134 // Keep in sync with the regex in dlang-bot: 135 // https://github.com/dlang/dlang-bot/blob/master/source/dlangbot/bugzilla.d#L29 136 // This regex was introduced in https://github.com/dlang/dlang-bot/pull/240 137 // and only the first part of the regex is needed (the second part matches 138 // issues reference that won't close the issue). 139 enum closedRE = ctRegex!(`(?:^fix(?:es)?(?:\s+(?:issues?|bugs?))?\s+(#?\d+(?:[\s,\+&and]+#?\d+)*))`, "i"); 140 141 auto issues = appender!(int[]); 142 foreach (repo; ["dmd", "druntime", "phobos", "dlang.org", "tools", "installer"] 143 .map!(r => buildPath("..", r))) 144 { 145 auto cmd = ["git", "-C", repo, "fetch", "--tags", "https://github.com/dlang/" ~ repo.baseName, 146 "+refs/heads/*:refs/remotes/upstream/*"]; 147 auto p = pipeProcess(cmd, Redirect.stdout); 148 enforce(wait(p.pid) == 0, "Failed to execute '%(%s %)'.".format(cmd)); 149 150 cmd = ["git", "-C", repo, "log", revRange]; 151 p = pipeProcess(cmd, Redirect.stdout); 152 scope(exit) enforce(wait(p.pid) == 0, "Failed to execute '%(%s %)'.".format(cmd)); 153 154 foreach (line; p.stdout.byLine()) 155 { 156 if (auto m = match(line.stripLeft, closedRE)) 157 { 158 m.captures[1] 159 .splitter(ctRegex!`[^\d]+`) 160 .filter!(b => b.length) 161 .map!(to!int) 162 .copy(issues); 163 } 164 } 165 } 166 return issues.data.sort().release.uniq; 167 } 168 169 /** Generate and return the change log as a string. */ 170 auto getBugzillaChanges(string revRange) 171 { 172 // component (e.g. DMD) -> bug type (e.g. regression) -> list of bug entries 173 BugzillaEntry[][string][string] entries; 174 175 auto issues = getIssues(revRange); 176 // abort prematurely if no issues are found in all git logs 177 if (issues.empty) 178 return entries; 179 180 auto req = generateRequest(templateRequest, issues); 181 debug stderr.writeln(req); // write text 182 auto data = req.get; 183 184 foreach (fields; csvReader!(Tuple!(int, string, string, string))(data, null)) 185 { 186 string comp = fields[1].toLower; 187 switch (comp) 188 { 189 case "dlang.org": comp = "dlang.org"; break; 190 case "dmd": comp = "DMD Compiler"; break; 191 case "druntime": comp = "Druntime"; break; 192 case "installer": comp = "Installer"; break; 193 case "phobos": comp = "Phobos"; break; 194 case "tools": comp = "Tools"; break; 195 case "dub": comp = "Dub"; break; 196 case "visuald": comp = "VisualD"; break; 197 default: assert(0, comp); 198 } 199 200 string type = fields[2].toLower; 201 switch (type) 202 { 203 case "regression": 204 type = "regression fixes"; 205 break; 206 207 case "blocker", "critical", "major", "normal", "minor", "trivial": 208 type = "bug fixes"; 209 break; 210 211 case "enhancement": 212 type = "enhancements"; 213 break; 214 215 default: assert(0, type); 216 } 217 218 auto entry = BugzillaEntry(fields[0], fields[3].idup); 219 entries[comp][type] ~= entry; 220 changelogStats.addBugzillaIssue(entry, comp, type); 221 } 222 return entries; 223 } 224 225 /** 226 Reads a single changelog file. 227 228 An entry consists of a title line, a blank separator line and 229 the description 230 231 Params: 232 filename = changelog file to be parsed 233 repoName = origin repository that contains the changelog entry 234 235 Returns: The parsed `ChangelogEntry` 236 */ 237 ChangelogEntry readChangelog(string filename, string repoName) 238 { 239 import std.algorithm.searching : countUntil; 240 import std.file : read; 241 import std.path : baseName, stripExtension; 242 import std.string : strip; 243 244 auto lines = filename.readText().splitLines(); 245 246 // filter empty files 247 if (lines.empty) 248 return ChangelogEntry.init; 249 250 // filter ddoc files 251 if (lines[0].startsWith("Ddoc")) 252 return ChangelogEntry.init; 253 254 enforce(lines.length >= 3 && 255 !lines[0].empty && 256 lines[1].empty && 257 !lines[2].empty, 258 "Changelog entries should consist of one title line, a blank separator line, and a description."); 259 260 ChangelogEntry entry = { 261 title: lines[0].strip, 262 description: lines[2..$].join("\n").strip, 263 basename: filename.baseName.stripExtension, 264 repo: repoName, 265 filePath: filename.findSplitAfter(repoName)[1].findSplitAfter("/")[1], 266 }; 267 return entry; 268 } 269 270 /** 271 Looks for changelog files (ending with `.dd`) in a directory and parses them. 272 273 Params: 274 changelogDir = directory to search for changelog files 275 repoName = origin repository that contains the changelog entry 276 277 Returns: An InputRange of `ChangelogEntry`s 278 */ 279 auto readTextChanges(string changelogDir, string repoName) 280 { 281 import std.algorithm.iteration : filter, map; 282 import std.file : dirEntries, SpanMode; 283 import std.string : endsWith; 284 285 return dirEntries(changelogDir, SpanMode.shallow) 286 .filter!(a => a.name().endsWith(".dd")) 287 .array.sort() 288 .map!(a => readChangelog(a, repoName)) 289 .filter!(a => a.title.length > 0); 290 } 291 292 /** 293 Writes the overview headline of the manually listed changes in the ddoc format as list. 294 295 Params: 296 changes = parsed InputRange of changelog information 297 w = Output range to use 298 */ 299 void writeTextChangesHeader(Entries, Writer)(Entries changes, Writer w, string headline) 300 if (isInputRange!Entries && isOutputRange!(Writer, string)) 301 { 302 // write the overview titles 303 w.formattedWrite("$(BUGSTITLE_TEXT_HEADER %s,\n\n", headline); 304 scope(exit) w.put("\n)\n\n"); 305 foreach(change; changes) 306 { 307 w.formattedWrite("$(LI $(RELATIVE_LINK2 %s,%s))\n", change.basename, change.title); 308 } 309 } 310 /** 311 Writes the long description of the manually listed changes in the ddoc format as list. 312 313 Params: 314 changes = parsed InputRange of changelog information 315 w = Output range to use 316 */ 317 void writeTextChangesBody(Entries, Writer)(Entries changes, Writer w, string headline) 318 if (isInputRange!Entries && isOutputRange!(Writer, string)) 319 { 320 w.formattedWrite("$(BUGSTITLE_TEXT_BODY %s,\n\n", headline); 321 scope(exit) w.put("\n)\n\n"); 322 foreach(change; changes) 323 { 324 w.formattedWrite("$(LI $(LNAME2 %s,%s)\n", change.basename, change.title); 325 w.formattedWrite("$(CHANGELOG_SOURCE_FILE %s, %s)\n", change.repo, change.filePath); 326 scope(exit) w.put(")\n\n"); 327 328 bool inPara, inCode; 329 foreach (line; change.description.splitLines) 330 { 331 if (line.stripLeft.startsWith("---", "```")) 332 { 333 if (inPara) 334 { 335 w.put(")\n"); 336 inPara = false; 337 } 338 inCode = !inCode; 339 } 340 else if (!inCode && !inPara && !line.empty) 341 { 342 w.put("$(P\n"); 343 inPara = true; 344 } 345 else if (inPara && line.empty) 346 { 347 w.put(")\n"); 348 inPara = false; 349 } 350 w.put(line); 351 w.put("\n"); 352 } 353 if (inPara) 354 w.put(")\n"); 355 } 356 } 357 358 /** 359 Writes the fixed issued from Bugzilla in the ddoc format as a single list. 360 361 Params: 362 changes = parsed InputRange of changelog information 363 w = Output range to use 364 */ 365 void writeBugzillaChanges(Entries, Writer)(Entries entries, Writer w) 366 if (isOutputRange!(Writer, string)) 367 { 368 immutable components = ["DMD Compiler", "Phobos", "Druntime", "dlang.org", "Optlink", "Tools", "Installer"]; 369 immutable bugtypes = ["regression fixes", "bug fixes", "enhancements"]; 370 371 foreach (component; components) 372 { 373 if (auto comp = component in entries) 374 { 375 foreach (bugtype; bugtypes) 376 if (auto bugs = bugtype in *comp) 377 { 378 w.formattedWrite("$(BUGSTITLE_BUGZILLA %s %s,\n\n", component, bugtype); 379 foreach (bug; sort!"a.id < b.id"(*bugs)) 380 { 381 w.formattedWrite("$(LI $(BUGZILLA %s): %s)\n", 382 bug.id, bug.summary.escapeParens()); 383 } 384 w.put(")\n"); 385 } 386 } 387 } 388 } 389 390 int main(string[] args) 391 { 392 auto outputFile = "./changelog.dd"; 393 auto nextVersionString = "LATEST"; 394 395 auto currDate = Clock.currTime(); 396 auto nextVersionDate = "%s %02d, %04d" 397 .format(currDate.month.to!string.capitalize, currDate.day, currDate.year); 398 399 string previousVersion = "Previous version"; 400 bool hideTextChanges = false; 401 string revRange; 402 403 auto helpInformation = getopt( 404 args, 405 std.getopt.config.passThrough, 406 "output|o", &outputFile, 407 "date", &nextVersionDate, 408 "version", &nextVersionString, 409 "prev-version", &previousVersion, // this can automatically be detected 410 "no-text", &hideTextChanges); 411 412 if (helpInformation.helpWanted) 413 { 414 `Changelog generator 415 Please supply a bugzilla version 416 ./changed.d "v2.071.2..upstream/stable"`.defaultGetoptPrinter(helpInformation.options); 417 } 418 419 if (args.length >= 2) 420 { 421 revRange = args[1]; 422 423 // extract the previous version 424 auto parts = revRange.split(".."); 425 if (parts.length > 1) 426 previousVersion = parts[0].replace("v", ""); 427 } 428 else 429 { 430 writeln("Skipped querying Bugzilla for changes. Please define a revision range e.g ./changed v2.072.2..upstream/stable"); 431 } 432 433 // location of the changelog files 434 alias Repo = Tuple!(string, "name", string, "headline", string, "path"); 435 auto repos = [Repo("dmd", "Compiler changes", null), 436 Repo("druntime", "Runtime changes", null), 437 Repo("phobos", "Library changes", null), 438 Repo("dlang.org", "Language changes", null), 439 Repo("installer", "Installer changes", null), 440 Repo("tools", "Tools changes", null), 441 Repo("dub", "Dub changes", null)]; 442 443 auto changedRepos = repos 444 .map!(repo => Repo(repo.name, repo.headline, buildPath(__FILE_FULL_PATH__.dirName, "..", repo.name, repo.name == "dlang.org" ? "language-changelog" : "changelog"))) 445 .filter!(r => r.path.exists); 446 447 // ensure that all files either end on .dd or .md 448 bool errors; 449 foreach (repo; changedRepos) 450 { 451 auto invalidFiles = repo.path 452 .dirEntries(SpanMode.shallow) 453 .filter!(a => !a.name.endsWith(".dd", ".md")); 454 if (!invalidFiles.empty) 455 { 456 invalidFiles.each!(f => stderr.writefln("ERROR: %s needs to have .dd or .md as extension", f.buildNormalizedPath)); 457 errors = 1; 458 } 459 } 460 import core.stdc.stdlib : exit; 461 if (errors) 462 1.exit; 463 464 auto f = File(outputFile, "w"); 465 auto w = f.lockingTextWriter(); 466 w.put("Ddoc\n\n"); 467 w.put("$(CHANGELOG_NAV_INJECT)\n\n"); 468 469 // Accumulate Bugzilla issues 470 typeof(revRange.getBugzillaChanges) bugzillaChanges; 471 if (revRange.length) 472 bugzillaChanges = revRange.getBugzillaChanges; 473 474 // Accumulate contributors from the git log 475 version(Contributors_Lib) 476 { 477 import contributors : FindConfig, findAuthors, reduceAuthors; 478 typeof(revRange.findAuthors(FindConfig.init).reduceAuthors.array) authors; 479 if (revRange) 480 { 481 FindConfig config = { 482 cwd: __FILE_FULL_PATH__.dirName.asNormalizedPath.to!string, 483 }; 484 config.mailmapFile = config.cwd.buildPath(".mailmap"); 485 authors = revRange.findAuthors(config).reduceAuthors.array; 486 changelogStats.contributors = authors.save.walkLength; 487 } 488 } 489 490 { 491 w.formattedWrite("$(VERSION %s, =================================================,\n\n", nextVersionDate); 492 493 scope(exit) w.put(")\n"); 494 495 496 if (!hideTextChanges) 497 { 498 // search for raw change files 499 auto changelogDirs = changedRepos 500 .map!(r => tuple!("headline", "changes")(r.headline, r.path.readTextChanges(r.name).array)) 501 .filter!(r => !r.changes.empty); 502 503 // accumulate stats 504 { 505 changelogDirs.each!(c => c.changes.each!(c => changelogStats.addChangelogEntry(c))); 506 w.put("$(CHANGELOG_HEADER_STATISTICS\n"); 507 scope(exit) w.put(")\n\n"); 508 509 with(changelogStats) 510 { 511 auto changelog = changelogEntries > 0 ? "%d major change%s and".format(changelogEntries, changelogEntries > 1 ? "s" : "") : ""; 512 w.put("$(VER) comes with {changelogEntries} {bugzillaIssues} fixed Bugzilla issue{bugzillaIssuesPlural}. 513 A huge thanks goes to the 514 $(LINK2 #contributors, {nrContributors} contributor{nrContributorsPlural}) 515 who made $(VER) possible." 516 .replace("{bugzillaIssues}", bugzillaIssues.text) 517 .replace("{bugzillaIssuesPlural}", bugzillaIssues != 1 ? "s" : "") 518 .replace("{changelogEntries}", changelog) 519 .replace("{nrContributors}", contributors.text) 520 .replace("{nrContributorsPlural}", contributors != 1 ? "s" : "") 521 ); 522 } 523 } 524 525 // print the overview headers 526 changelogDirs.each!(c => c.changes.writeTextChangesHeader(w, c.headline)); 527 528 if (!revRange.empty) 529 w.put("$(CHANGELOG_SEP_HEADER_TEXT_NONEMPTY)\n\n"); 530 531 w.put("$(CHANGELOG_SEP_HEADER_TEXT)\n\n"); 532 533 // print the detailed descriptions 534 changelogDirs.each!(x => x.changes.writeTextChangesBody(w, x.headline)); 535 536 if (revRange.length) 537 w.put("$(CHANGELOG_SEP_TEXT_BUGZILLA)\n\n"); 538 } 539 else 540 { 541 w.put("$(CHANGELOG_SEP_NO_TEXT_BUGZILLA)\n\n"); 542 } 543 544 // print the entire changelog history 545 if (revRange.length) 546 bugzillaChanges.writeBugzillaChanges(w); 547 } 548 549 version(Contributors_Lib) 550 if (revRange) 551 { 552 w.formattedWrite("$(D_CONTRIBUTORS_HEADER %d)\n", changelogStats.contributors); 553 w.put("$(D_CONTRIBUTORS\n"); 554 authors.each!(a => w.formattedWrite(" $(D_CONTRIBUTOR %s)\n", a.name)); 555 w.put(")\n"); 556 w.put("$(D_CONTRIBUTORS_FOOTER)\n"); 557 } 558 559 w.put("$(CHANGELOG_NAV_INJECT)\n\n"); 560 561 // write own macros 562 w.formattedWrite(`Macros: 563 VER=%s 564 TITLE=Change Log: $(VER) 565 `, nextVersionString); 566 567 writefln("Change log generated to: '%s'", outputFile); 568 return 0; 569 }