1 #!/usr/bin/env rdmd
2 //          Copyright Martin Nowak 2012.
3 // Distributed under the Boost Software License, Version 1.0.
4 //    (See accompanying file LICENSE_1_0.txt or copy at
5 //          http://www.boost.org/LICENSE_1_0.txt)
6 module dget;
7 
8 import std.algorithm, std.exception, std.file, std.range, std.net.curl;
9 static import std.stdio;
10 
11 void usage()
12 {
13     std.stdio.writeln("usage: dget [--clone|-c] [--help|-h] <repo>...");
14 }
15 
16 int main(string[] args)
17 {
18     if (args.length == 1) { usage(); return 1; }
19 
20     import std.getopt;
21     bool clone, help;
22     getopt(args,
23            "clone|c", &clone,
24            "help|h", &help);
25 
26     if (help) { usage(); return 0; }
27 
28     import std.typetuple;
29     string user, repo;
30     foreach(arg; args[1 .. $])
31     {
32         TypeTuple!(user, repo) = resolveRepo(arg);
33         enforce(!repo.exists, fmt("output folder '%s' already exists", repo));
34         if (clone) cloneRepo(user, repo);
35         else fetchMaster(user, repo).unzipTo(repo);
36     }
37     return 0;
38 }
39 
40 //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
41 
42 /// default github users for repo lookup
43 immutable defaultUsers = ["D-Programming-Deimos", "dlang"];
44 
45 auto resolveRepo(string arg)
46 {
47     import std.regex;
48 
49     enum rule = regex(r"^(?:([^/:]*)/)?([^/:]*)$");
50     auto m = match(arg, rule);
51     enforce(!m.empty, fmt("expected 'user/repo' but found '%s'", arg));
52 
53     auto user = m.captures[1];
54     auto repo = m.captures[2];
55 
56     if (user.empty)
57     {
58         auto tail = defaultUsers.find!(u => u.hasRepo(repo))();
59         enforce(!tail.empty, fmt("repo '%s' was not found among '%(%s, %)'",
60                                   repo, defaultUsers));
61         user = tail.front;
62     }
63     import std.typecons;
64     return tuple(user, repo);
65 }
66 
67 bool hasRepo(string user, string repo)
68 {
69     return fmt("https://api.github.com/users/%s/repos?type=public", user)
70         .reqJSON().array.canFind!(a => a.object["name"].str == repo)();
71 }
72 
73 //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
74 
75 void cloneRepo(string user, string repo)
76 {
77     import std.process;
78     enforce(executeShell(fmt("git clone git://github.com/%s/%s", user, repo)).status == 0);
79 }
80 
81 //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
82 
83 ubyte[] fetchMaster(string user, string repo)
84 {
85     auto url = fmt("https://api.github.com/repos/%s/%s/git/refs/heads/master", user, repo);
86     auto sha = url.reqJSON().object["object"].object["sha"].str;
87     std.stdio.writefln("fetching %s/%s@%s", user, repo, sha);
88     return download(fmt("https://github.com/%s/%s/zipball/master", user, repo));
89 }
90 
91 //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
92 
93 auto reqJSON(string url)
94 {
95     import std.json;
96     return parseJSON(get(url));
97 }
98 
99 //..............................................................................
100 
101 ubyte[] download(string url)
102 {
103     // doesn't work because it already timeouts after 2 minutes
104     // return get!(HTTP, ubyte)();
105 
106     import core.time, std.array, std.conv;
107 
108     auto buf = appender!(ubyte[])();
109     size_t contentLength;
110 
111     auto http = HTTP(url);
112     http.method = HTTP.Method.get;
113     http.onReceiveHeader((k, v)
114     {
115         if (k == "content-length")
116             contentLength = to!size_t(v);
117     });
118     http.onReceive((data)
119     {
120         buf.put(data);
121         std.stdio.writef("%sk/%sk\r", buf.data.length/1024,
122                          contentLength ? to!string(contentLength/1024) : "?");
123         std.stdio.stdout.flush();
124         return data.length;
125     });
126     http.dataTimeout = dur!"msecs"(0);
127     http.perform();
128     immutable sc = http.statusLine().code;
129     enforce(sc / 100 == 2 || sc == 302,
130             fmt("HTTP request returned status code %s", sc));
131     std.stdio.writeln("done                    ");
132     return buf.data;
133 }
134 
135 //..............................................................................
136 
137 void unzipTo(ubyte[] data, string outdir)
138 {
139     import std.path, std..string, std.zip;
140 
141     scope archive = new ZipArchive(data);
142     std.stdio.writeln("unpacking:");
143     string prefix;
144     mkdir(outdir);
145 
146     foreach(name, _; archive.directory)
147     {
148         prefix = name[0 .. $ - name.find("/").length + 1];
149         break;
150     }
151     foreach(name, am; archive.directory)
152     {
153         if (!am.expandedSize) continue;
154 
155         string path = buildPath(outdir, chompPrefix(name, prefix));
156         std.stdio.writeln(path);
157         auto dir = dirName(path);
158         if (!dir.empty && !dir.exists)
159             mkdirRecurse(dir);
160         archive.expand(am);
161         std.file.write(path, am.expandedData);
162     }
163 }
164 
165 //..............................................................................
166 
167 string fmt(Args...)(string fmt, auto ref Args args)
168 {
169     import std.array, std.format;
170     auto app = appender!string();
171     formattedWrite(app, fmt, args);
172     return app.data;
173 }