1 //          Copyright Brian Schott (Hackerpilot) 2012.
2 // Distributed under the Boost Software License, Version 1.0.
3 //    (See accompanying file LICENSE_1_0.txt or copy at
4 //          http://www.boost.org/LICENSE_1_0.txt)
5 
6 module dscanner.ctags;
7 
8 import dparse.parser;
9 import dparse.lexer;
10 import dparse.ast;
11 import dparse.rollback_allocator;
12 import std.algorithm;
13 import std.range;
14 import std.stdio;
15 import std.array;
16 import std.conv;
17 import std.typecons;
18 import containers.ttree;
19 import std..string;
20 import std.functional : toDelegate;
21 
22 /**
23  * Prints CTAGS information to the given file.
24  * Includes metadata according to exuberant format used by Vim.
25  * Params:
26  *     output = the file that Exuberant TAGS info is written to
27  *     fileNames = tags will be generated from these files
28  */
29 void printCtags(File output, string[] fileNames)
30 {
31 	TTree!string tags;
32 	LexerConfig config;
33 	StringCache cache = StringCache(StringCache.defaultBucketCount);
34 	foreach (fileName; fileNames)
35 	{
36 		RollbackAllocator rba;
37 
38 		ubyte[] bytes;
39 
40 		if (fileName == "stdin")
41 		{
42 			File f = std.stdio.stdin;
43 			immutable stepSize = 4096;
44 			ubyte[] buffer = uninitializedArray!(ubyte[])(stepSize);
45 
46 			while (true)
47 			{
48 				auto dataRead = f.rawRead(buffer[$ - stepSize .. $]);
49 				if (dataRead.length < stepSize)
50 				{
51 					bytes = buffer[0 .. $ - (stepSize - dataRead.length)];
52 					break;
53 				}
54 				buffer.length += stepSize;
55 			}
56 		}
57 		else
58 		{
59 			File f = File(fileName);
60 
61 			if (f.size == 0)
62 				continue;
63 
64 			bytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
65 			f.rawRead(bytes);
66 		}
67 
68 		auto tokens = getTokensForParser(bytes, config, &cache);
69 		Module m = parseModule(tokens.array, fileName, &rba, toDelegate(&doNothing));
70 
71 		auto printer = new CTagsPrinter(&tags);
72 		printer.fileName = fileName;
73 		printer.visit(m);
74 	}
75 	output.write("!_TAG_FILE_FORMAT\t2\n" ~ "!_TAG_FILE_SORTED\t1\n" ~ "!_TAG_FILE_AUTHOR\tBrian Schott\n"
76 			~ "!_TAG_PROGRAM_URL\thttps://github.com/Hackerpilot/Dscanner/\n");
77 	tags[].copy(output.lockingTextWriter);
78 }
79 
80 private:
81 
82 /// States determining how an access modifier affects tags when traversing the
83 /// AST.
84 /// The assumption is that there are fewer AST nodes and patterns that affects
85 /// the whole scope.
86 /// Therefor the default was chosen to be Reset.
87 enum AccessState
88 {
89 	Reset, /// when ascending the AST reset back to the previous access.
90 	Keep /// when ascending the AST keep the new access.
91 }
92 
93 alias ContextType = Tuple!(string, "c", string, "access");
94 
95 void doNothing(string, size_t, size_t, string, bool)
96 {
97 }
98 
99 string paramsToString(Dec)(const Dec dec)
100 {
101 	import dparse.formatter : Formatter;
102 
103 	auto app = appender!string();
104 	auto formatter = new Formatter!(typeof(app))(app);
105 
106 	static if (is(Dec == FunctionDeclaration) || is(Dec == Constructor))
107 	{
108 		formatter.format(dec.parameters);
109 	}
110 	else static if (is(Dec == TemplateDeclaration))
111 	{
112 		formatter.format(dec.templateParameters);
113 	}
114 
115 	return app.data;
116 }
117 
118 final class CTagsPrinter : ASTVisitor
119 {
120 	public this(TTree!string* output)
121 	{
122 		this.tagLines = output;
123 	}
124 
125 	override void visit(const ClassDeclaration dec)
126 	{
127 		tagLines.insert("%s\t%s\t%d;\"\tc\tline:%d%s%s\n".format(dec.name.text,
128 				fileName, dec.name.line, dec.name.line, context.c, context.access));
129 		immutable c = context;
130 		context = ContextType("\tclass:" ~ dec.name.text, "\taccess:public");
131 		dec.accept(this);
132 		context = c;
133 	}
134 
135 	override void visit(const StructDeclaration dec)
136 	{
137 		if (dec.name == tok!"")
138 		{
139 			dec.accept(this);
140 			return;
141 		}
142 		tagLines.insert("%s\t%s\t%d;\"\ts\tline:%d%s%s\n".format(dec.name.text,
143 				fileName, dec.name.line, dec.name.line, context.c, context.access));
144 		immutable c = context;
145 		context = ContextType("\tstruct:" ~ dec.name.text, "\taccess:public");
146 		dec.accept(this);
147 		context = c;
148 	}
149 
150 	override void visit(const InterfaceDeclaration dec)
151 	{
152 		tagLines.insert("%s\t%s\t%d;\"\ti\tline:%d%s%s\n".format(dec.name.text,
153 				fileName, dec.name.line, dec.name.line, context.c, context.access));
154 		immutable c = context;
155 		context = ContextType("\tinterface:" ~ dec.name.text, context.access);
156 		dec.accept(this);
157 		context = c;
158 	}
159 
160 	override void visit(const TemplateDeclaration dec)
161 	{
162 		auto params = paramsToString(dec);
163 
164 		tagLines.insert("%s\t%s\t%d;\"\tT\tline:%d%s%s\tsignature:%s\n".format(dec.name.text,
165 				fileName, dec.name.line, dec.name.line, context.c, context.access, params));
166 		immutable c = context;
167 		context = ContextType("\ttemplate:" ~ dec.name.text, context.access);
168 		dec.accept(this);
169 		context = c;
170 	}
171 
172 	override void visit(const FunctionDeclaration dec)
173 	{
174 		auto params = paramsToString(dec);
175 
176 		tagLines.insert("%s\t%s\t%d;\"\tf\tline:%d%s%s\tsignature:%s\n".format(dec.name.text,
177 				fileName, dec.name.line, dec.name.line, context.c, context.access, params));
178 	}
179 
180 	override void visit(const Constructor dec)
181 	{
182 		auto params = paramsToString(dec);
183 
184 		tagLines.insert("this\t%s\t%d;\"\tf\tline:%d%s\tsignature:%s%s\n".format(fileName,
185 				dec.line, dec.line, context.c, params, context.access));
186 	}
187 
188 	override void visit(const Destructor dec)
189 	{
190 		tagLines.insert("~this\t%s\t%d;\"\tf\tline:%d%s\n".format(fileName,
191 				dec.line, dec.line, context.c));
192 	}
193 
194 	override void visit(const EnumDeclaration dec)
195 	{
196 		tagLines.insert("%s\t%s\t%d;\"\tg\tline:%d%s%s\n".format(dec.name.text,
197 				fileName, dec.name.line, dec.name.line, context.c, context.access));
198 		immutable c = context;
199 		context = ContextType("\tenum:" ~ dec.name.text, context.access);
200 		dec.accept(this);
201 		context = c;
202 	}
203 
204 	override void visit(const UnionDeclaration dec)
205 	{
206 		if (dec.name == tok!"")
207 		{
208 			dec.accept(this);
209 			return;
210 		}
211 		tagLines.insert("%s\t%s\t%d;\"\tu\tline:%d%s\n".format(dec.name.text,
212 				fileName, dec.name.line, dec.name.line, context.c));
213 		immutable c = context;
214 		context = ContextType("\tunion:" ~ dec.name.text, context.access);
215 		dec.accept(this);
216 		context = c;
217 	}
218 
219 	override void visit(const AnonymousEnumMember mem)
220 	{
221 		tagLines.insert("%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text,
222 				fileName, mem.name.line, mem.name.line, context.c));
223 	}
224 
225 	override void visit(const EnumMember mem)
226 	{
227 		tagLines.insert("%s\t%s\t%d;\"\te\tline:%d%s\n".format(mem.name.text,
228 				fileName, mem.name.line, mem.name.line, context.c));
229 	}
230 
231 	override void visit(const VariableDeclaration dec)
232 	{
233 		foreach (d; dec.declarators)
234 		{
235 			tagLines.insert("%s\t%s\t%d;\"\tv\tline:%d%s%s\n".format(d.name.text,
236 					fileName, d.name.line, d.name.line, context.c, context.access));
237 		}
238 		dec.accept(this);
239 	}
240 
241 	override void visit(const AutoDeclaration dec)
242 	{
243 		foreach (i; dec.parts.map!(a => a.identifier))
244 		{
245 			tagLines.insert("%s\t%s\t%d;\"\tv\tline:%d%s%s\n".format(i.text,
246 					fileName, i.line, i.line, context.c, context.access));
247 		}
248 		dec.accept(this);
249 	}
250 
251 	override void visit(const Invariant dec)
252 	{
253 		tagLines.insert("invariant\t%s\t%d;\"\tv\tline:%d%s%s\n".format(fileName,
254 				dec.line, dec.line, context.c, context.access));
255 	}
256 
257 	override void visit(const ModuleDeclaration dec)
258 	{
259 		context = ContextType("", "\taccess:public");
260 		dec.accept(this);
261 	}
262 
263 	override void visit(const Attribute attribute)
264 	{
265 		if (attribute.attribute != tok!"")
266 		{
267 			switch (attribute.attribute.type)
268 			{
269 			case tok!"export":
270 				context.access = "\taccess:public";
271 				break;
272 			case tok!"public":
273 				context.access = "\taccess:public";
274 				break;
275 			case tok!"package":
276 				context.access = "\taccess:protected";
277 				break;
278 			case tok!"protected":
279 				context.access = "\taccess:protected";
280 				break;
281 			case tok!"private":
282 				context.access = "\taccess:private";
283 				break;
284 			default:
285 			}
286 		}
287 
288 		attribute.accept(this);
289 	}
290 
291 	override void visit(const AttributeDeclaration dec)
292 	{
293 		accessSt = AccessState.Keep;
294 		dec.accept(this);
295 	}
296 
297 	override void visit(const Declaration dec)
298 	{
299 		immutable c = context;
300 		dec.accept(this);
301 
302 		final switch (accessSt) with (AccessState)
303 		{
304 		case Reset:
305 			context = c;
306 			break;
307 		case Keep:
308 			break;
309 		}
310 		accessSt = AccessState.Reset;
311 	}
312 
313 	override void visit(const Unittest dec)
314 	{
315 		// skipping symbols inside a unit test to not clutter the ctags file
316 		// with "temporary" symbols.
317 		// TODO when phobos have a unittest library investigate how that could
318 		// be used to describe the tests.
319 		// Maybe with UDA's to give the unittest a "name".
320 	}
321 
322 	override void visit(const AliasDeclaration dec)
323 	{
324 		// Old style alias
325 		if (dec.declaratorIdentifierList)
326 		{
327 			foreach (i; dec.declaratorIdentifierList.identifiers)
328 			{
329 				tagLines.insert("%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(i.text,
330 						fileName, i.line, i.line, context.c, context.access));
331 			}
332 		}
333 		dec.accept(this);
334 	}
335 
336 	override void visit(const AliasInitializer dec)
337 	{
338 		tagLines.insert("%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(dec.name.text,
339 				fileName, dec.name.line, dec.name.line, context.c, context.access));
340 
341 		dec.accept(this);
342 	}
343 
344 	override void visit(const AliasThisDeclaration dec)
345 	{
346 		auto name = dec.identifier;
347 		tagLines.insert("%s\t%s\t%d;\"\ta\tline:%d%s%s\n".format(name.text,
348 				fileName, name.line, name.line, context.c, context.access));
349 
350 		dec.accept(this);
351 	}
352 
353 	alias visit = ASTVisitor.visit;
354 	string fileName;
355 	TTree!string* tagLines;
356 	ContextType context;
357 	AccessState accessSt;
358 }