Python code coverage for Parser/parsetok.c

#	count	content
1	n/a
2	n/a	/* Parser-tokenizer link implementation */
3	n/a
4	n/a	#include "pgenheaders.h"
5	n/a	#include "tokenizer.h"
6	n/a	#include "node.h"
7	n/a	#include "grammar.h"
8	n/a	#include "parser.h"
9	n/a	#include "parsetok.h"
10	n/a	#include "errcode.h"
11	n/a	#include "graminit.h"
12	n/a
13	n/a
14	n/a	/* Forward */
15	n/a	static node parsetok(struct tok_state , grammar , int, perrdetail , int *);
16	n/a	static int initerr(perrdetail err_ret, PyObject filename);
17	n/a
18	n/a	/* Parse input coming from a string. Return error code, print some errors. */
19	n/a	node *
20	n/a	PyParser_ParseString(const char s, grammar g, int start, perrdetail *err_ret)
21	n/a	{
22	n/a	return PyParser_ParseStringFlagsFilename(s, NULL, g, start, err_ret, 0);
23	n/a	}
24	n/a
25	n/a	node *
26	n/a	PyParser_ParseStringFlags(const char s, grammar g, int start,
27	n/a	perrdetail *err_ret, int flags)
28	n/a	{
29	n/a	return PyParser_ParseStringFlagsFilename(s, NULL,
30	n/a	g, start, err_ret, flags);
31	n/a	}
32	n/a
33	n/a	node *
34	n/a	PyParser_ParseStringFlagsFilename(const char s, const char filename,
35	n/a	grammar *g, int start,
36	n/a	perrdetail *err_ret, int flags)
37	n/a	{
38	n/a	int iflags = flags;
39	n/a	return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
40	n/a	err_ret, &iflags);
41	n/a	}
42	n/a
43	n/a	node *
44	n/a	PyParser_ParseStringObject(const char s, PyObject filename,
45	n/a	grammar *g, int start,
46	n/a	perrdetail err_ret, int flags)
47	n/a	{
48	n/a	struct tok_state *tok;
49	n/a	int exec_input = start == file_input;
50	n/a
51	n/a	if (initerr(err_ret, filename) < 0)
52	n/a	return NULL;
53	n/a
54	n/a	if (*flags & PyPARSE_IGNORE_COOKIE)
55	n/a	tok = PyTokenizer_FromUTF8(s, exec_input);
56	n/a	else
57	n/a	tok = PyTokenizer_FromString(s, exec_input);
58	n/a	if (tok == NULL) {
59	n/a	err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
60	n/a	return NULL;
61	n/a	}
62	n/a
63	n/a	#ifndef PGEN
64	n/a	Py_INCREF(err_ret->filename);
65	n/a	tok->filename = err_ret->filename;
66	n/a	#endif
67	n/a	return parsetok(tok, g, start, err_ret, flags);
68	n/a	}
69	n/a
70	n/a	node *
71	n/a	PyParser_ParseStringFlagsFilenameEx(const char s, const char filename_str,
72	n/a	grammar *g, int start,
73	n/a	perrdetail err_ret, int flags)
74	n/a	{
75	n/a	node *n;
76	n/a	PyObject *filename = NULL;
77	n/a	#ifndef PGEN
78	n/a	if (filename_str != NULL) {
79	n/a	filename = PyUnicode_DecodeFSDefault(filename_str);
80	n/a	if (filename == NULL) {
81	n/a	err_ret->error = E_ERROR;
82	n/a	return NULL;
83	n/a	}
84	n/a	}
85	n/a	#endif
86	n/a	n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags);
87	n/a	#ifndef PGEN
88	n/a	Py_XDECREF(filename);
89	n/a	#endif
90	n/a	return n;
91	n/a	}
92	n/a
93	n/a	/* Parse input coming from a file. Return error code, print some errors. */
94	n/a
95	n/a	node *
96	n/a	PyParser_ParseFile(FILE fp, const char filename, grammar *g, int start,
97	n/a	const char ps1, const char ps2,
98	n/a	perrdetail *err_ret)
99	n/a	{
100	n/a	return PyParser_ParseFileFlags(fp, filename, NULL,
101	n/a	g, start, ps1, ps2, err_ret, 0);
102	n/a	}
103	n/a
104	n/a	node *
105	n/a	PyParser_ParseFileFlags(FILE fp, const char filename, const char *enc,
106	n/a	grammar *g, int start,
107	n/a	const char ps1, const char ps2,
108	n/a	perrdetail *err_ret, int flags)
109	n/a	{
110	n/a	int iflags = flags;
111	n/a	return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
112	n/a	ps2, err_ret, &iflags);
113	n/a	}
114	n/a
115	n/a	node *
116	n/a	PyParser_ParseFileObject(FILE fp, PyObject filename,
117	n/a	const char enc, grammar g, int start,
118	n/a	const char ps1, const char ps2,
119	n/a	perrdetail err_ret, int flags)
120	n/a	{
121	n/a	struct tok_state *tok;
122	n/a
123	n/a	if (initerr(err_ret, filename) < 0)
124	n/a	return NULL;
125	n/a
126	n/a	if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
127	n/a	err_ret->error = E_NOMEM;
128	n/a	return NULL;
129	n/a	}
130	n/a	#ifndef PGEN
131	n/a	Py_INCREF(err_ret->filename);
132	n/a	tok->filename = err_ret->filename;
133	n/a	#endif
134	n/a	return parsetok(tok, g, start, err_ret, flags);
135	n/a	}
136	n/a
137	n/a	node *
138	n/a	PyParser_ParseFileFlagsEx(FILE fp, const char filename,
139	n/a	const char enc, grammar g, int start,
140	n/a	const char ps1, const char ps2,
141	n/a	perrdetail err_ret, int flags)
142	n/a	{
143	n/a	node *n;
144	n/a	PyObject *fileobj = NULL;
145	n/a	#ifndef PGEN
146	n/a	if (filename != NULL) {
147	n/a	fileobj = PyUnicode_DecodeFSDefault(filename);
148	n/a	if (fileobj == NULL) {
149	n/a	err_ret->error = E_ERROR;
150	n/a	return NULL;
151	n/a	}
152	n/a	}
153	n/a	#endif
154	n/a	n = PyParser_ParseFileObject(fp, fileobj, enc, g,
155	n/a	start, ps1, ps2, err_ret, flags);
156	n/a	#ifndef PGEN
157	n/a	Py_XDECREF(fileobj);
158	n/a	#endif
159	n/a	return n;
160	n/a	}
161	n/a
162	n/a	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
163	n/a	#if 0
164	n/a	static const char with_msg[] =
165	n/a	"%s:%d: Warning: 'with' will become a reserved keyword in Python 2.6\n";
166	n/a
167	n/a	static const char as_msg[] =
168	n/a	"%s:%d: Warning: 'as' will become a reserved keyword in Python 2.6\n";
169	n/a
170	n/a	static void
171	n/a	warn(const char msg, const char filename, int lineno)
172	n/a	{
173	n/a	if (filename == NULL)
174	n/a	filename = "<string>";
175	n/a	PySys_WriteStderr(msg, filename, lineno);
176	n/a	}
177	n/a	#endif
178	n/a	#endif
179	n/a
180	n/a	/* Parse input coming from the given tokenizer structure.
181	n/a	Return error code. */
182	n/a
183	n/a	static node *
184	n/a	parsetok(struct tok_state tok, grammar g, int start, perrdetail *err_ret,
185	n/a	int *flags)
186	n/a	{
187	n/a	parser_state *ps;
188	n/a	node *n;
189	n/a	int started = 0;
190	n/a
191	n/a	if ((ps = PyParser_New(g, start)) == NULL) {
192	n/a	err_ret->error = E_NOMEM;
193	n/a	PyTokenizer_Free(tok);
194	n/a	return NULL;
195	n/a	}
196	n/a	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
197	n/a	if (*flags & PyPARSE_BARRY_AS_BDFL)
198	n/a	ps->p_flags \|= CO_FUTURE_BARRY_AS_BDFL;
199	n/a	#endif
200	n/a
201	n/a	for (;;) {
202	n/a	char a, b;
203	n/a	int type;
204	n/a	size_t len;
205	n/a	char *str;
206	n/a	int col_offset;
207	n/a
208	n/a	type = PyTokenizer_Get(tok, &a, &b);
209	n/a	if (type == ERRORTOKEN) {
210	n/a	err_ret->error = tok->done;
211	n/a	break;
212	n/a	}
213	n/a	if (type == ENDMARKER && started) {
214	n/a	type = NEWLINE; /* Add an extra newline */
215	n/a	started = 0;
216	n/a	/* Add the right number of dedent tokens,
217	n/a	except if a certain flag is given --
218	n/a	codeop.py uses this. */
219	n/a	if (tok->indent &&
220	n/a	!(*flags & PyPARSE_DONT_IMPLY_DEDENT))
221	n/a	{
222	n/a	tok->pendin = -tok->indent;
223	n/a	tok->indent = 0;
224	n/a	}
225	n/a	}
226	n/a	else
227	n/a	started = 1;
228	n/a	len = b - a; /* XXX this may compute NULL - NULL */
229	n/a	str = (char *) PyObject_MALLOC(len + 1);
230	n/a	if (str == NULL) {
231	n/a	err_ret->error = E_NOMEM;
232	n/a	break;
233	n/a	}
234	n/a	if (len > 0)
235	n/a	strncpy(str, a, len);
236	n/a	str[len] = '\0';
237	n/a
238	n/a	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
239	n/a	if (type == NOTEQUAL) {
240	n/a	if (!(ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
241	n/a	strcmp(str, "!=")) {
242	n/a	PyObject_FREE(str);
243	n/a	err_ret->error = E_SYNTAX;
244	n/a	break;
245	n/a	}
246	n/a	else if ((ps->p_flags & CO_FUTURE_BARRY_AS_BDFL) &&
247	n/a	strcmp(str, "<>")) {
248	n/a	PyObject_FREE(str);
249	n/a	err_ret->text = "with Barry as BDFL, use '<>' "
250	n/a	"instead of '!='";
251	n/a	err_ret->error = E_SYNTAX;
252	n/a	break;
253	n/a	}
254	n/a	}
255	n/a	#endif
256	n/a	if (a >= tok->line_start)
257	n/a	col_offset = Py_SAFE_DOWNCAST(a - tok->line_start,
258	n/a	intptr_t, int);
259	n/a	else
260	n/a	col_offset = -1;
261	n/a
262	n/a	if ((err_ret->error =
263	n/a	PyParser_AddToken(ps, (int)type, str,
264	n/a	tok->lineno, col_offset,
265	n/a	&(err_ret->expected))) != E_OK) {
266	n/a	if (err_ret->error != E_DONE) {
267	n/a	PyObject_FREE(str);
268	n/a	err_ret->token = type;
269	n/a	}
270	n/a	break;
271	n/a	}
272	n/a	}
273	n/a
274	n/a	if (err_ret->error == E_DONE) {
275	n/a	n = ps->p_tree;
276	n/a	ps->p_tree = NULL;
277	n/a
278	n/a	#ifndef PGEN
279	n/a	/* Check that the source for a single input statement really
280	n/a	is a single statement by looking at what is left in the
281	n/a	buffer after parsing. Trailing whitespace and comments
282	n/a	are OK. */
283	n/a	if (start == single_input) {
284	n/a	char *cur = tok->cur;
285	n/a	char c = *tok->cur;
286	n/a
287	n/a	for (;;) {
288	n/a	while (c == ' ' \|\| c == '\t' \|\| c == '\n' \|\| c == '\014')
289	n/a	c = *++cur;
290	n/a
291	n/a	if (!c)
292	n/a	break;
293	n/a
294	n/a	if (c != '#') {
295	n/a	err_ret->error = E_BADSINGLE;
296	n/a	PyNode_Free(n);
297	n/a	n = NULL;
298	n/a	break;
299	n/a	}
300	n/a
301	n/a	/* Suck up comment. */
302	n/a	while (c && c != '\n')
303	n/a	c = *++cur;
304	n/a	}
305	n/a	}
306	n/a	#endif
307	n/a	}
308	n/a	else
309	n/a	n = NULL;
310	n/a
311	n/a	#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
312	n/a	*flags = ps->p_flags;
313	n/a	#endif
314	n/a	PyParser_Delete(ps);
315	n/a
316	n/a	if (n == NULL) {
317	n/a	if (tok->done == E_EOF)
318	n/a	err_ret->error = E_EOF;
319	n/a	err_ret->lineno = tok->lineno;
320	n/a	if (tok->buf != NULL) {
321	n/a	size_t len;
322	n/a	assert(tok->cur - tok->buf < INT_MAX);
323	n/a	err_ret->offset = (int)(tok->cur - tok->buf);
324	n/a	len = tok->inp - tok->buf;
325	n/a	err_ret->text = (char *) PyObject_MALLOC(len + 1);
326	n/a	if (err_ret->text != NULL) {
327	n/a	if (len > 0)
328	n/a	strncpy(err_ret->text, tok->buf, len);
329	n/a	err_ret->text[len] = '\0';
330	n/a	}
331	n/a	}
332	n/a	} else if (tok->encoding != NULL) {
333	n/a	/* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
334	n/a	* allocated using PyMem_
335	n/a	*/
336	n/a	node* r = PyNode_New(encoding_decl);
337	n/a	if (r)
338	n/a	r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
339	n/a	if (!r \|\| !r->n_str) {
340	n/a	err_ret->error = E_NOMEM;
341	n/a	if (r)
342	n/a	PyObject_FREE(r);
343	n/a	n = NULL;
344	n/a	goto done;
345	n/a	}
346	n/a	strcpy(r->n_str, tok->encoding);
347	n/a	PyMem_FREE(tok->encoding);
348	n/a	tok->encoding = NULL;
349	n/a	r->n_nchildren = 1;
350	n/a	r->n_child = n;
351	n/a	n = r;
352	n/a	}
353	n/a
354	n/a	done:
355	n/a	PyTokenizer_Free(tok);
356	n/a
357	n/a	return n;
358	n/a	}
359	n/a
360	n/a	static int
361	n/a	initerr(perrdetail err_ret, PyObject filename)
362	n/a	{
363	n/a	err_ret->error = E_OK;
364	n/a	err_ret->lineno = 0;
365	n/a	err_ret->offset = 0;
366	n/a	err_ret->text = NULL;
367	n/a	err_ret->token = -1;
368	n/a	err_ret->expected = -1;
369	n/a	#ifndef PGEN
370	n/a	if (filename) {
371	n/a	Py_INCREF(filename);
372	n/a	err_ret->filename = filename;
373	n/a	}
374	n/a	else {
375	n/a	err_ret->filename = PyUnicode_FromString("<string>");
376	n/a	if (err_ret->filename == NULL) {
377	n/a	err_ret->error = E_ERROR;
378	n/a	return -1;
379	n/a	}
380	n/a	}
381	n/a	#endif
382	n/a	return 0;
383	n/a	}