1 #!/usr/bin/env rdmd 2 // Copyright Martin Nowak 2012. 3 // Distributed under the Boost Software License, Version 1.0. 4 // (See accompanying file LICENSE_1_0.txt or copy at 5 // http://www.boost.org/LICENSE_1_0.txt) 6 module dget; 7 8 import std.algorithm, std.exception, std.file, std.range, std.net.curl; 9 static import std.stdio; 10 11 void usage() 12 { 13 std.stdio.writeln("usage: dget [--clone|-c] [--help|-h] <repo>..."); 14 } 15 16 int main(string[] args) 17 { 18 if (args.length == 1) { usage(); return 1; } 19 20 import std.getopt; 21 bool clone, help; 22 getopt(args, 23 "clone|c", &clone, 24 "help|h", &help); 25 26 if (help) { usage(); return 0; } 27 28 import std.typetuple; 29 string user, repo; 30 foreach(arg; args[1 .. $]) 31 { 32 TypeTuple!(user, repo) = resolveRepo(arg); 33 enforce(!repo.exists, fmt("output folder '%s' already exists", repo)); 34 if (clone) cloneRepo(user, repo); 35 else fetchMaster(user, repo).unzipTo(repo); 36 } 37 return 0; 38 } 39 40 //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: 41 42 /// default github users for repo lookup 43 immutable defaultUsers = ["D-Programming-Deimos", "dlang"]; 44 45 auto resolveRepo(string arg) 46 { 47 import std.regex; 48 49 enum rule = regex(r"^(?:([^/:]*)/)?([^/:]*)$"); 50 auto m = match(arg, rule); 51 enforce(!m.empty, fmt("expected 'user/repo' but found '%s'", arg)); 52 53 auto user = m.captures[1]; 54 auto repo = m.captures[2]; 55 56 if (user.empty) 57 { 58 auto tail = defaultUsers.find!(u => u.hasRepo(repo))(); 59 enforce(!tail.empty, fmt("repo '%s' was not found among '%(%s, %)'", 60 repo, defaultUsers)); 61 user = tail.front; 62 } 63 import std.typecons; 64 return tuple(user, repo); 65 } 66 67 bool hasRepo(string user, string repo) 68 { 69 return fmt("https://api.github.com/users/%s/repos?type=public", user) 70 .reqJSON().array.canFind!(a => a.object["name"].str == repo)(); 71 } 72 73 //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: 74 75 void cloneRepo(string user, string repo) 76 { 77 import std.process; 78 enforce(executeShell(fmt("git clone git://github.com/%s/%s", user, repo)).status == 0); 79 } 80 81 //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: 82 83 ubyte[] fetchMaster(string user, string repo) 84 { 85 auto url = fmt("https://api.github.com/repos/%s/%s/git/refs/heads/master", user, repo); 86 auto sha = url.reqJSON().object["object"].object["sha"].str; 87 std.stdio.writefln("fetching %s/%s@%s", user, repo, sha); 88 return download(fmt("https://github.com/%s/%s/zipball/master", user, repo)); 89 } 90 91 //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: 92 93 auto reqJSON(string url) 94 { 95 import std.json; 96 return parseJSON(get(url)); 97 } 98 99 //.............................................................................. 100 101 ubyte[] download(string url) 102 { 103 // doesn't work because it already timeouts after 2 minutes 104 // return get!(HTTP, ubyte)(); 105 106 import core.time, std.array, std.conv; 107 108 auto buf = appender!(ubyte[])(); 109 size_t contentLength; 110 111 auto http = HTTP(url); 112 http.method = HTTP.Method.get; 113 http.onReceiveHeader((k, v) 114 { 115 if (k == "content-length") 116 contentLength = to!size_t(v); 117 }); 118 http.onReceive((data) 119 { 120 buf.put(data); 121 std.stdio.writef("%sk/%sk\r", buf.data.length/1024, 122 contentLength ? to!string(contentLength/1024) : "?"); 123 std.stdio.stdout.flush(); 124 return data.length; 125 }); 126 http.dataTimeout = dur!"msecs"(0); 127 http.perform(); 128 immutable sc = http.statusLine().code; 129 enforce(sc / 100 == 2 || sc == 302, 130 fmt("HTTP request returned status code %s", sc)); 131 std.stdio.writeln("done "); 132 return buf.data; 133 } 134 135 //.............................................................................. 136 137 void unzipTo(ubyte[] data, string outdir) 138 { 139 import std.path, std..string, std.zip; 140 141 scope archive = new ZipArchive(data); 142 std.stdio.writeln("unpacking:"); 143 string prefix; 144 mkdir(outdir); 145 146 foreach(name, _; archive.directory) 147 { 148 prefix = name[0 .. $ - name.find("/").length + 1]; 149 break; 150 } 151 foreach(name, am; archive.directory) 152 { 153 if (!am.expandedSize) continue; 154 155 string path = buildPath(outdir, chompPrefix(name, prefix)); 156 std.stdio.writeln(path); 157 auto dir = dirName(path); 158 if (!dir.empty && !dir.exists) 159 mkdirRecurse(dir); 160 archive.expand(am); 161 std.file.write(path, am.expandedData); 162 } 163 } 164 165 //.............................................................................. 166 167 string fmt(Args...)(string fmt, auto ref Args args) 168 { 169 import std.array, std.format; 170 auto app = appender!string(); 171 formattedWrite(app, fmt, args); 172 return app.data; 173 }