Python code coverage for Modules/_ctypes/libffi/src/aarch64/ffi.c

#	count	content
1	n/a	/* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
2	n/a
3	n/a	Permission is hereby granted, free of charge, to any person obtaining
4	n/a	a copy of this software and associated documentation files (the
5	n/a	``Software''), to deal in the Software without restriction, including
6	n/a	without limitation the rights to use, copy, modify, merge, publish,
7	n/a	distribute, sublicense, and/or sell copies of the Software, and to
8	n/a	permit persons to whom the Software is furnished to do so, subject to
9	n/a	the following conditions:
10	n/a
11	n/a	The above copyright notice and this permission notice shall be
12	n/a	included in all copies or substantial portions of the Software.
13	n/a
14	n/a	THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
15	n/a	EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16	n/a	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17	n/a	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18	n/a	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19	n/a	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20	n/a	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
21	n/a
22	n/a	#include <stdio.h>
23	n/a
24	n/a	#include <ffi.h>
25	n/a	#include <ffi_common.h>
26	n/a
27	n/a	#include <stdlib.h>
28	n/a
29	n/a	/* Stack alignment requirement in bytes */
30	n/a	#if defined (__APPLE__)
31	n/a	#define AARCH64_STACK_ALIGN 1
32	n/a	#else
33	n/a	#define AARCH64_STACK_ALIGN 16
34	n/a	#endif
35	n/a
36	n/a	#define N_X_ARG_REG 8
37	n/a	#define N_V_ARG_REG 8
38	n/a
39	n/a	#define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
40	n/a
41	n/a	union _d
42	n/a	{
43	n/a	UINT64 d;
44	n/a	UINT32 s[2];
45	n/a	};
46	n/a
47	n/a	struct call_context
48	n/a	{
49	n/a	UINT64 x [AARCH64_N_XREG];
50	n/a	struct
51	n/a	{
52	n/a	union _d d[2];
53	n/a	} v [AARCH64_N_VREG];
54	n/a	};
55	n/a
56	n/a	#if defined (__clang__) && defined (__APPLE__)
57	n/a	extern void
58	n/a	sys_icache_invalidate (void *start, size_t len);
59	n/a	#endif
60	n/a
61	n/a	static inline void
62	n/a	ffi_clear_cache (void start, void end)
63	n/a	{
64	n/a	#if defined (__clang__) && defined (__APPLE__)
65	n/a	sys_icache_invalidate (start, (char )end - (char )start);
66	n/a	#elif defined (__GNUC__)
67	n/a	__builtin___clear_cache (start, end);
68	n/a	#else
69	n/a	#error "Missing builtin to flush instruction cache"
70	n/a	#endif
71	n/a	}
72	n/a
73	n/a	static void *
74	n/a	get_x_addr (struct call_context *context, unsigned n)
75	n/a	{
76	n/a	return &context->x[n];
77	n/a	}
78	n/a
79	n/a	static void *
80	n/a	get_s_addr (struct call_context *context, unsigned n)
81	n/a	{
82	n/a	#if defined __AARCH64EB__
83	n/a	return &context->v[n].d[1].s[1];
84	n/a	#else
85	n/a	return &context->v[n].d[0].s[0];
86	n/a	#endif
87	n/a	}
88	n/a
89	n/a	static void *
90	n/a	get_d_addr (struct call_context *context, unsigned n)
91	n/a	{
92	n/a	#if defined __AARCH64EB__
93	n/a	return &context->v[n].d[1];
94	n/a	#else
95	n/a	return &context->v[n].d[0];
96	n/a	#endif
97	n/a	}
98	n/a
99	n/a	static void *
100	n/a	get_v_addr (struct call_context *context, unsigned n)
101	n/a	{
102	n/a	return &context->v[n];
103	n/a	}
104	n/a
105	n/a	/* Return the memory location at which a basic type would reside
106	n/a	were it to have been stored in register n. */
107	n/a
108	n/a	static void *
109	n/a	get_basic_type_addr (unsigned short type, struct call_context *context,
110	n/a	unsigned n)
111	n/a	{
112	n/a	switch (type)
113	n/a	{
114	n/a	case FFI_TYPE_FLOAT:
115	n/a	return get_s_addr (context, n);
116	n/a	case FFI_TYPE_DOUBLE:
117	n/a	return get_d_addr (context, n);
118	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
119	n/a	case FFI_TYPE_LONGDOUBLE:
120	n/a	return get_v_addr (context, n);
121	n/a	#endif
122	n/a	case FFI_TYPE_UINT8:
123	n/a	case FFI_TYPE_SINT8:
124	n/a	case FFI_TYPE_UINT16:
125	n/a	case FFI_TYPE_SINT16:
126	n/a	case FFI_TYPE_UINT32:
127	n/a	case FFI_TYPE_SINT32:
128	n/a	case FFI_TYPE_INT:
129	n/a	case FFI_TYPE_POINTER:
130	n/a	case FFI_TYPE_UINT64:
131	n/a	case FFI_TYPE_SINT64:
132	n/a	return get_x_addr (context, n);
133	n/a	case FFI_TYPE_VOID:
134	n/a	return NULL;
135	n/a	default:
136	n/a	FFI_ASSERT (0);
137	n/a	return NULL;
138	n/a	}
139	n/a	}
140	n/a
141	n/a	/* Return the alignment width for each of the basic types. */
142	n/a
143	n/a	static size_t
144	n/a	get_basic_type_alignment (unsigned short type)
145	n/a	{
146	n/a	switch (type)
147	n/a	{
148	n/a	case FFI_TYPE_FLOAT:
149	n/a	case FFI_TYPE_DOUBLE:
150	n/a	return sizeof (UINT64);
151	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
152	n/a	case FFI_TYPE_LONGDOUBLE:
153	n/a	return sizeof (long double);
154	n/a	#endif
155	n/a	case FFI_TYPE_UINT8:
156	n/a	case FFI_TYPE_SINT8:
157	n/a	#if defined (__APPLE__)
158	n/a	return sizeof (UINT8);
159	n/a	#endif
160	n/a	case FFI_TYPE_UINT16:
161	n/a	case FFI_TYPE_SINT16:
162	n/a	#if defined (__APPLE__)
163	n/a	return sizeof (UINT16);
164	n/a	#endif
165	n/a	case FFI_TYPE_UINT32:
166	n/a	case FFI_TYPE_INT:
167	n/a	case FFI_TYPE_SINT32:
168	n/a	#if defined (__APPLE__)
169	n/a	return sizeof (UINT32);
170	n/a	#endif
171	n/a	case FFI_TYPE_POINTER:
172	n/a	case FFI_TYPE_UINT64:
173	n/a	case FFI_TYPE_SINT64:
174	n/a	return sizeof (UINT64);
175	n/a
176	n/a	default:
177	n/a	FFI_ASSERT (0);
178	n/a	return 0;
179	n/a	}
180	n/a	}
181	n/a
182	n/a	/* Return the size in bytes for each of the basic types. */
183	n/a
184	n/a	static size_t
185	n/a	get_basic_type_size (unsigned short type)
186	n/a	{
187	n/a	switch (type)
188	n/a	{
189	n/a	case FFI_TYPE_FLOAT:
190	n/a	return sizeof (UINT32);
191	n/a	case FFI_TYPE_DOUBLE:
192	n/a	return sizeof (UINT64);
193	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
194	n/a	case FFI_TYPE_LONGDOUBLE:
195	n/a	return sizeof (long double);
196	n/a	#endif
197	n/a	case FFI_TYPE_UINT8:
198	n/a	return sizeof (UINT8);
199	n/a	case FFI_TYPE_SINT8:
200	n/a	return sizeof (SINT8);
201	n/a	case FFI_TYPE_UINT16:
202	n/a	return sizeof (UINT16);
203	n/a	case FFI_TYPE_SINT16:
204	n/a	return sizeof (SINT16);
205	n/a	case FFI_TYPE_UINT32:
206	n/a	return sizeof (UINT32);
207	n/a	case FFI_TYPE_INT:
208	n/a	case FFI_TYPE_SINT32:
209	n/a	return sizeof (SINT32);
210	n/a	case FFI_TYPE_POINTER:
211	n/a	case FFI_TYPE_UINT64:
212	n/a	return sizeof (UINT64);
213	n/a	case FFI_TYPE_SINT64:
214	n/a	return sizeof (SINT64);
215	n/a
216	n/a	default:
217	n/a	FFI_ASSERT (0);
218	n/a	return 0;
219	n/a	}
220	n/a	}
221	n/a
222	n/a	extern void
223	n/a	ffi_call_SYSV (unsigned ()(struct call_context context, unsigned char *,
224	n/a	extended_cif *),
225	n/a	struct call_context *context,
226	n/a	extended_cif *,
227	n/a	size_t,
228	n/a	void (*fn)(void));
229	n/a
230	n/a	extern void
231	n/a	ffi_closure_SYSV (ffi_closure *);
232	n/a
233	n/a	/* Test for an FFI floating point representation. */
234	n/a
235	n/a	static unsigned
236	n/a	is_floating_type (unsigned short type)
237	n/a	{
238	n/a	return (type == FFI_TYPE_FLOAT \|\| type == FFI_TYPE_DOUBLE
239	n/a	\|\| type == FFI_TYPE_LONGDOUBLE);
240	n/a	}
241	n/a
242	n/a	/* Test for a homogeneous structure. */
243	n/a
244	n/a	static unsigned short
245	n/a	get_homogeneous_type (ffi_type *ty)
246	n/a	{
247	n/a	if (ty->type == FFI_TYPE_STRUCT && ty->elements)
248	n/a	{
249	n/a	unsigned i;
250	n/a	unsigned short candidate_type
251	n/a	= get_homogeneous_type (ty->elements[0]);
252	n/a	for (i =1; ty->elements[i]; i++)
253	n/a	{
254	n/a	unsigned short iteration_type = 0;
255	n/a	/* If we have a nested struct, we must find its homogeneous type.
256	n/a	If that fits with our candidate type, we are still
257	n/a	homogeneous. */
258	n/a	if (ty->elements[i]->type == FFI_TYPE_STRUCT
259	n/a	&& ty->elements[i]->elements)
260	n/a	{
261	n/a	iteration_type = get_homogeneous_type (ty->elements[i]);
262	n/a	}
263	n/a	else
264	n/a	{
265	n/a	iteration_type = ty->elements[i]->type;
266	n/a	}
267	n/a
268	n/a	/* If we are not homogeneous, return FFI_TYPE_STRUCT. */
269	n/a	if (candidate_type != iteration_type)
270	n/a	return FFI_TYPE_STRUCT;
271	n/a	}
272	n/a	return candidate_type;
273	n/a	}
274	n/a
275	n/a	/* Base case, we have no more levels of nesting, so we
276	n/a	are a basic type, and so, trivially homogeneous in that type. */
277	n/a	return ty->type;
278	n/a	}
279	n/a
280	n/a	/* Determine the number of elements within a STRUCT.
281	n/a
282	n/a	Note, we must handle nested structs.
283	n/a
284	n/a	If ty is not a STRUCT this function will return 0. */
285	n/a
286	n/a	static unsigned
287	n/a	element_count (ffi_type *ty)
288	n/a	{
289	n/a	if (ty->type == FFI_TYPE_STRUCT && ty->elements)
290	n/a	{
291	n/a	unsigned n;
292	n/a	unsigned elems = 0;
293	n/a	for (n = 0; ty->elements[n]; n++)
294	n/a	{
295	n/a	if (ty->elements[n]->type == FFI_TYPE_STRUCT
296	n/a	&& ty->elements[n]->elements)
297	n/a	elems += element_count (ty->elements[n]);
298	n/a	else
299	n/a	elems++;
300	n/a	}
301	n/a	return elems;
302	n/a	}
303	n/a	return 0;
304	n/a	}
305	n/a
306	n/a	/* Test for a homogeneous floating point aggregate.
307	n/a
308	n/a	A homogeneous floating point aggregate is a homogeneous aggregate of
309	n/a	a half- single- or double- precision floating point type with one
310	n/a	to four elements. Note that this includes nested structs of the
311	n/a	basic type. */
312	n/a
313	n/a	static int
314	n/a	is_hfa (ffi_type *ty)
315	n/a	{
316	n/a	if (ty->type == FFI_TYPE_STRUCT
317	n/a	&& ty->elements[0]
318	n/a	&& is_floating_type (get_homogeneous_type (ty)))
319	n/a	{
320	n/a	unsigned n = element_count (ty);
321	n/a	return n >= 1 && n <= 4;
322	n/a	}
323	n/a	return 0;
324	n/a	}
325	n/a
326	n/a	/* Test if an ffi_type is a candidate for passing in a register.
327	n/a
328	n/a	This test does not check that sufficient registers of the
329	n/a	appropriate class are actually available, merely that IFF
330	n/a	sufficient registers are available then the argument will be passed
331	n/a	in register(s).
332	n/a
333	n/a	Note that an ffi_type that is deemed to be a register candidate
334	n/a	will always be returned in registers.
335	n/a
336	n/a	Returns 1 if a register candidate else 0. */
337	n/a
338	n/a	static int
339	n/a	is_register_candidate (ffi_type *ty)
340	n/a	{
341	n/a	switch (ty->type)
342	n/a	{
343	n/a	case FFI_TYPE_VOID:
344	n/a	case FFI_TYPE_FLOAT:
345	n/a	case FFI_TYPE_DOUBLE:
346	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
347	n/a	case FFI_TYPE_LONGDOUBLE:
348	n/a	#endif
349	n/a	case FFI_TYPE_UINT8:
350	n/a	case FFI_TYPE_UINT16:
351	n/a	case FFI_TYPE_UINT32:
352	n/a	case FFI_TYPE_UINT64:
353	n/a	case FFI_TYPE_POINTER:
354	n/a	case FFI_TYPE_SINT8:
355	n/a	case FFI_TYPE_SINT16:
356	n/a	case FFI_TYPE_SINT32:
357	n/a	case FFI_TYPE_INT:
358	n/a	case FFI_TYPE_SINT64:
359	n/a	return 1;
360	n/a
361	n/a	case FFI_TYPE_STRUCT:
362	n/a	if (is_hfa (ty))
363	n/a	{
364	n/a	return 1;
365	n/a	}
366	n/a	else if (ty->size > 16)
367	n/a	{
368	n/a	/* Too large. Will be replaced with a pointer to memory. The
369	n/a	pointer MAY be passed in a register, but the value will
370	n/a	not. This test specifically fails since the argument will
371	n/a	never be passed by value in registers. */
372	n/a	return 0;
373	n/a	}
374	n/a	else
375	n/a	{
376	n/a	/* Might be passed in registers depending on the number of
377	n/a	registers required. */
378	n/a	return (ty->size + 7) / 8 < N_X_ARG_REG;
379	n/a	}
380	n/a	break;
381	n/a
382	n/a	default:
383	n/a	FFI_ASSERT (0);
384	n/a	break;
385	n/a	}
386	n/a
387	n/a	return 0;
388	n/a	}
389	n/a
390	n/a	/* Test if an ffi_type argument or result is a candidate for a vector
391	n/a	register. */
392	n/a
393	n/a	static int
394	n/a	is_v_register_candidate (ffi_type *ty)
395	n/a	{
396	n/a	return is_floating_type (ty->type)
397	n/a	\|\| (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
398	n/a	}
399	n/a
400	n/a	/* Representation of the procedure call argument marshalling
401	n/a	state.
402	n/a
403	n/a	The terse state variable names match the names used in the AARCH64
404	n/a	PCS. */
405	n/a
406	n/a	struct arg_state
407	n/a	{
408	n/a	unsigned ngrn; /* Next general-purpose register number. */
409	n/a	unsigned nsrn; /* Next vector register number. */
410	n/a	size_t nsaa; /* Next stack offset. */
411	n/a
412	n/a	#if defined (__APPLE__)
413	n/a	unsigned allocating_variadic;
414	n/a	#endif
415	n/a	};
416	n/a
417	n/a	/* Initialize a procedure call argument marshalling state. */
418	n/a	static void
419	n/a	arg_init (struct arg_state *state, size_t call_frame_size)
420	n/a	{
421	n/a	state->ngrn = 0;
422	n/a	state->nsrn = 0;
423	n/a	state->nsaa = 0;
424	n/a
425	n/a	#if defined (__APPLE__)
426	n/a	state->allocating_variadic = 0;
427	n/a	#endif
428	n/a	}
429	n/a
430	n/a	/* Return the number of available consecutive core argument
431	n/a	registers. */
432	n/a
433	n/a	static unsigned
434	n/a	available_x (struct arg_state *state)
435	n/a	{
436	n/a	return N_X_ARG_REG - state->ngrn;
437	n/a	}
438	n/a
439	n/a	/* Return the number of available consecutive vector argument
440	n/a	registers. */
441	n/a
442	n/a	static unsigned
443	n/a	available_v (struct arg_state *state)
444	n/a	{
445	n/a	return N_V_ARG_REG - state->nsrn;
446	n/a	}
447	n/a
448	n/a	static void *
449	n/a	allocate_to_x (struct call_context context, struct arg_state state)
450	n/a	{
451	n/a	FFI_ASSERT (state->ngrn < N_X_ARG_REG);
452	n/a	return get_x_addr (context, (state->ngrn)++);
453	n/a	}
454	n/a
455	n/a	static void *
456	n/a	allocate_to_s (struct call_context context, struct arg_state state)
457	n/a	{
458	n/a	FFI_ASSERT (state->nsrn < N_V_ARG_REG);
459	n/a	return get_s_addr (context, (state->nsrn)++);
460	n/a	}
461	n/a
462	n/a	static void *
463	n/a	allocate_to_d (struct call_context context, struct arg_state state)
464	n/a	{
465	n/a	FFI_ASSERT (state->nsrn < N_V_ARG_REG);
466	n/a	return get_d_addr (context, (state->nsrn)++);
467	n/a	}
468	n/a
469	n/a	static void *
470	n/a	allocate_to_v (struct call_context context, struct arg_state state)
471	n/a	{
472	n/a	FFI_ASSERT (state->nsrn < N_V_ARG_REG);
473	n/a	return get_v_addr (context, (state->nsrn)++);
474	n/a	}
475	n/a
476	n/a	/* Allocate an aligned slot on the stack and return a pointer to it. */
477	n/a	static void *
478	n/a	allocate_to_stack (struct arg_state state, void stack, size_t alignment,
479	n/a	size_t size)
480	n/a	{
481	n/a	void *allocation;
482	n/a
483	n/a	/* Round up the NSAA to the larger of 8 or the natural
484	n/a	alignment of the argument's type. */
485	n/a	state->nsaa = ALIGN (state->nsaa, alignment);
486	n/a	state->nsaa = ALIGN (state->nsaa, alignment);
487	n/a	#if defined (__APPLE__)
488	n/a	if (state->allocating_variadic)
489	n/a	state->nsaa = ALIGN (state->nsaa, 8);
490	n/a	#else
491	n/a	state->nsaa = ALIGN (state->nsaa, 8);
492	n/a	#endif
493	n/a
494	n/a	allocation = stack + state->nsaa;
495	n/a
496	n/a	state->nsaa += size;
497	n/a	return allocation;
498	n/a	}
499	n/a
500	n/a	static void
501	n/a	copy_basic_type (void dest, void source, unsigned short type)
502	n/a	{
503	n/a	/* This is necessary to ensure that basic types are copied
504	n/a	sign extended to 64-bits as libffi expects. */
505	n/a	switch (type)
506	n/a	{
507	n/a	case FFI_TYPE_FLOAT:
508	n/a	(float ) dest = (float ) source;
509	n/a	break;
510	n/a	case FFI_TYPE_DOUBLE:
511	n/a	(double ) dest = (double ) source;
512	n/a	break;
513	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
514	n/a	case FFI_TYPE_LONGDOUBLE:
515	n/a	(long double ) dest = (long double ) source;
516	n/a	break;
517	n/a	#endif
518	n/a	case FFI_TYPE_UINT8:
519	n/a	(ffi_arg ) dest = (UINT8 ) source;
520	n/a	break;
521	n/a	case FFI_TYPE_SINT8:
522	n/a	(ffi_sarg ) dest = (SINT8 ) source;
523	n/a	break;
524	n/a	case FFI_TYPE_UINT16:
525	n/a	(ffi_arg ) dest = (UINT16 ) source;
526	n/a	break;
527	n/a	case FFI_TYPE_SINT16:
528	n/a	(ffi_sarg ) dest = (SINT16 ) source;
529	n/a	break;
530	n/a	case FFI_TYPE_UINT32:
531	n/a	(ffi_arg ) dest = (UINT32 ) source;
532	n/a	break;
533	n/a	case FFI_TYPE_INT:
534	n/a	case FFI_TYPE_SINT32:
535	n/a	(ffi_sarg ) dest = (SINT32 ) source;
536	n/a	break;
537	n/a	case FFI_TYPE_POINTER:
538	n/a	case FFI_TYPE_UINT64:
539	n/a	(ffi_arg ) dest = (UINT64 ) source;
540	n/a	break;
541	n/a	case FFI_TYPE_SINT64:
542	n/a	(ffi_sarg ) dest = (SINT64 ) source;
543	n/a	break;
544	n/a	case FFI_TYPE_VOID:
545	n/a	break;
546	n/a
547	n/a	default:
548	n/a	FFI_ASSERT (0);
549	n/a	}
550	n/a	}
551	n/a
552	n/a	static void
553	n/a	copy_hfa_to_reg_or_stack (void *memory,
554	n/a	ffi_type *ty,
555	n/a	struct call_context *context,
556	n/a	unsigned char *stack,
557	n/a	struct arg_state *state)
558	n/a	{
559	n/a	unsigned elems = element_count (ty);
560	n/a	if (available_v (state) < elems)
561	n/a	{
562	n/a	/* There are insufficient V registers. Further V register allocations
563	n/a	are prevented, the NSAA is adjusted (by allocate_to_stack ())
564	n/a	and the argument is copied to memory at the adjusted NSAA. */
565	n/a	state->nsrn = N_V_ARG_REG;
566	n/a	memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size),
567	n/a	memory,
568	n/a	ty->size);
569	n/a	}
570	n/a	else
571	n/a	{
572	n/a	int i;
573	n/a	unsigned short type = get_homogeneous_type (ty);
574	n/a	for (i = 0; i < elems; i++)
575	n/a	{
576	n/a	void *reg = allocate_to_v (context, state);
577	n/a	copy_basic_type (reg, memory, type);
578	n/a	memory += get_basic_type_size (type);
579	n/a	}
580	n/a	}
581	n/a	}
582	n/a
583	n/a	/* Either allocate an appropriate register for the argument type, or if
584	n/a	none are available, allocate a stack slot and return a pointer
585	n/a	to the allocated space. */
586	n/a
587	n/a	static void *
588	n/a	allocate_to_register_or_stack (struct call_context *context,
589	n/a	unsigned char *stack,
590	n/a	struct arg_state *state,
591	n/a	unsigned short type)
592	n/a	{
593	n/a	size_t alignment = get_basic_type_alignment (type);
594	n/a	size_t size = alignment;
595	n/a	switch (type)
596	n/a	{
597	n/a	case FFI_TYPE_FLOAT:
598	n/a	/* This is the only case for which the allocated stack size
599	n/a	should not match the alignment of the type. */
600	n/a	size = sizeof (UINT32);
601	n/a	/* Fall through. */
602	n/a	case FFI_TYPE_DOUBLE:
603	n/a	if (state->nsrn < N_V_ARG_REG)
604	n/a	return allocate_to_d (context, state);
605	n/a	state->nsrn = N_V_ARG_REG;
606	n/a	break;
607	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
608	n/a	case FFI_TYPE_LONGDOUBLE:
609	n/a	if (state->nsrn < N_V_ARG_REG)
610	n/a	return allocate_to_v (context, state);
611	n/a	state->nsrn = N_V_ARG_REG;
612	n/a	break;
613	n/a	#endif
614	n/a	case FFI_TYPE_UINT8:
615	n/a	case FFI_TYPE_SINT8:
616	n/a	case FFI_TYPE_UINT16:
617	n/a	case FFI_TYPE_SINT16:
618	n/a	case FFI_TYPE_UINT32:
619	n/a	case FFI_TYPE_SINT32:
620	n/a	case FFI_TYPE_INT:
621	n/a	case FFI_TYPE_POINTER:
622	n/a	case FFI_TYPE_UINT64:
623	n/a	case FFI_TYPE_SINT64:
624	n/a	if (state->ngrn < N_X_ARG_REG)
625	n/a	return allocate_to_x (context, state);
626	n/a	state->ngrn = N_X_ARG_REG;
627	n/a	break;
628	n/a	default:
629	n/a	FFI_ASSERT (0);
630	n/a	}
631	n/a
632	n/a	return allocate_to_stack (state, stack, alignment, size);
633	n/a	}
634	n/a
635	n/a	/* Copy a value to an appropriate register, or if none are
636	n/a	available, to the stack. */
637	n/a
638	n/a	static void
639	n/a	copy_to_register_or_stack (struct call_context *context,
640	n/a	unsigned char *stack,
641	n/a	struct arg_state *state,
642	n/a	void *value,
643	n/a	unsigned short type)
644	n/a	{
645	n/a	copy_basic_type (
646	n/a	allocate_to_register_or_stack (context, stack, state, type),
647	n/a	value,
648	n/a	type);
649	n/a	}
650	n/a
651	n/a	/* Marshall the arguments from FFI representation to procedure call
652	n/a	context and stack. */
653	n/a
654	n/a	static unsigned
655	n/a	aarch64_prep_args (struct call_context context, unsigned char stack,
656	n/a	extended_cif *ecif)
657	n/a	{
658	n/a	int i;
659	n/a	struct arg_state state;
660	n/a
661	n/a	arg_init (&state, ALIGN(ecif->cif->bytes, 16));
662	n/a
663	n/a	for (i = 0; i < ecif->cif->nargs; i++)
664	n/a	{
665	n/a	ffi_type *ty = ecif->cif->arg_types[i];
666	n/a	switch (ty->type)
667	n/a	{
668	n/a	case FFI_TYPE_VOID:
669	n/a	FFI_ASSERT (0);
670	n/a	break;
671	n/a
672	n/a	/* If the argument is a basic type the argument is allocated to an
673	n/a	appropriate register, or if none are available, to the stack. */
674	n/a	case FFI_TYPE_FLOAT:
675	n/a	case FFI_TYPE_DOUBLE:
676	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
677	n/a	case FFI_TYPE_LONGDOUBLE:
678	n/a	#endif
679	n/a	case FFI_TYPE_UINT8:
680	n/a	case FFI_TYPE_SINT8:
681	n/a	case FFI_TYPE_UINT16:
682	n/a	case FFI_TYPE_SINT16:
683	n/a	case FFI_TYPE_UINT32:
684	n/a	case FFI_TYPE_INT:
685	n/a	case FFI_TYPE_SINT32:
686	n/a	case FFI_TYPE_POINTER:
687	n/a	case FFI_TYPE_UINT64:
688	n/a	case FFI_TYPE_SINT64:
689	n/a	copy_to_register_or_stack (context, stack, &state,
690	n/a	ecif->avalue[i], ty->type);
691	n/a	break;
692	n/a
693	n/a	case FFI_TYPE_STRUCT:
694	n/a	if (is_hfa (ty))
695	n/a	{
696	n/a	copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context,
697	n/a	stack, &state);
698	n/a	}
699	n/a	else if (ty->size > 16)
700	n/a	{
701	n/a	/* If the argument is a composite type that is larger than 16
702	n/a	bytes, then the argument has been copied to memory, and
703	n/a	the argument is replaced by a pointer to the copy. */
704	n/a
705	n/a	copy_to_register_or_stack (context, stack, &state,
706	n/a	&(ecif->avalue[i]), FFI_TYPE_POINTER);
707	n/a	}
708	n/a	else if (available_x (&state) >= (ty->size + 7) / 8)
709	n/a	{
710	n/a	/* If the argument is a composite type and the size in
711	n/a	double-words is not more than the number of available
712	n/a	X registers, then the argument is copied into consecutive
713	n/a	X registers. */
714	n/a	int j;
715	n/a	for (j = 0; j < (ty->size + 7) / 8; j++)
716	n/a	{
717	n/a	memcpy (allocate_to_x (context, &state),
718	n/a	&(((UINT64 *) ecif->avalue[i])[j]),
719	n/a	sizeof (UINT64));
720	n/a	}
721	n/a	}
722	n/a	else
723	n/a	{
724	n/a	/* Otherwise, there are insufficient X registers. Further X
725	n/a	register allocations are prevented, the NSAA is adjusted
726	n/a	(by allocate_to_stack ()) and the argument is copied to
727	n/a	memory at the adjusted NSAA. */
728	n/a	state.ngrn = N_X_ARG_REG;
729	n/a
730	n/a	memcpy (allocate_to_stack (&state, stack, ty->alignment,
731	n/a	ty->size), ecif->avalue + i, ty->size);
732	n/a	}
733	n/a	break;
734	n/a
735	n/a	default:
736	n/a	FFI_ASSERT (0);
737	n/a	break;
738	n/a	}
739	n/a
740	n/a	#if defined (__APPLE__)
741	n/a	if (i + 1 == ecif->cif->aarch64_nfixedargs)
742	n/a	{
743	n/a	state.ngrn = N_X_ARG_REG;
744	n/a	state.nsrn = N_V_ARG_REG;
745	n/a
746	n/a	state.allocating_variadic = 1;
747	n/a	}
748	n/a	#endif
749	n/a	}
750	n/a
751	n/a	return ecif->cif->aarch64_flags;
752	n/a	}
753	n/a
754	n/a	ffi_status
755	n/a	ffi_prep_cif_machdep (ffi_cif *cif)
756	n/a	{
757	n/a	/* Round the stack up to a multiple of the stack alignment requirement. */
758	n/a	cif->bytes =
759	n/a	(cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1);
760	n/a
761	n/a	/* Initialize our flags. We are interested if this CIF will touch a
762	n/a	vector register, if so we will enable context save and load to
763	n/a	those registers, otherwise not. This is intended to be friendly
764	n/a	to lazy float context switching in the kernel. */
765	n/a	cif->aarch64_flags = 0;
766	n/a
767	n/a	if (is_v_register_candidate (cif->rtype))
768	n/a	{
769	n/a	cif->aarch64_flags \|= AARCH64_FFI_WITH_V;
770	n/a	}
771	n/a	else
772	n/a	{
773	n/a	int i;
774	n/a	for (i = 0; i < cif->nargs; i++)
775	n/a	if (is_v_register_candidate (cif->arg_types[i]))
776	n/a	{
777	n/a	cif->aarch64_flags \|= AARCH64_FFI_WITH_V;
778	n/a	break;
779	n/a	}
780	n/a	}
781	n/a
782	n/a	return FFI_OK;
783	n/a	}
784	n/a
785	n/a	#if defined (__APPLE__)
786	n/a
787	n/a	/* Perform Apple-specific cif processing for variadic calls */
788	n/a	ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif,
789	n/a	unsigned int nfixedargs,
790	n/a	unsigned int ntotalargs)
791	n/a	{
792	n/a	cif->aarch64_nfixedargs = nfixedargs;
793	n/a
794	n/a	return ffi_prep_cif_machdep(cif);
795	n/a	}
796	n/a
797	n/a	#endif
798	n/a
799	n/a	/* Call a function with the provided arguments and capture the return
800	n/a	value. */
801	n/a	void
802	n/a	ffi_call (ffi_cif cif, void (fn)(void), void rvalue, void *avalue)
803	n/a	{
804	n/a	extended_cif ecif;
805	n/a
806	n/a	ecif.cif = cif;
807	n/a	ecif.avalue = avalue;
808	n/a	ecif.rvalue = rvalue;
809	n/a
810	n/a	switch (cif->abi)
811	n/a	{
812	n/a	case FFI_SYSV:
813	n/a	{
814	n/a	struct call_context context;
815	n/a	size_t stack_bytes;
816	n/a
817	n/a	/* Figure out the total amount of stack space we need, the
818	n/a	above call frame space needs to be 16 bytes aligned to
819	n/a	ensure correct alignment of the first object inserted in
820	n/a	that space hence the ALIGN applied to cif->bytes.*/
821	n/a	stack_bytes = ALIGN(cif->bytes, 16);
822	n/a
823	n/a	memset (&context, 0, sizeof (context));
824	n/a	if (is_register_candidate (cif->rtype))
825	n/a	{
826	n/a	ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
827	n/a	switch (cif->rtype->type)
828	n/a	{
829	n/a	case FFI_TYPE_VOID:
830	n/a	case FFI_TYPE_FLOAT:
831	n/a	case FFI_TYPE_DOUBLE:
832	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
833	n/a	case FFI_TYPE_LONGDOUBLE:
834	n/a	#endif
835	n/a	case FFI_TYPE_UINT8:
836	n/a	case FFI_TYPE_SINT8:
837	n/a	case FFI_TYPE_UINT16:
838	n/a	case FFI_TYPE_SINT16:
839	n/a	case FFI_TYPE_UINT32:
840	n/a	case FFI_TYPE_SINT32:
841	n/a	case FFI_TYPE_POINTER:
842	n/a	case FFI_TYPE_UINT64:
843	n/a	case FFI_TYPE_INT:
844	n/a	case FFI_TYPE_SINT64:
845	n/a	{
846	n/a	void *addr = get_basic_type_addr (cif->rtype->type,
847	n/a	&context, 0);
848	n/a	copy_basic_type (rvalue, addr, cif->rtype->type);
849	n/a	break;
850	n/a	}
851	n/a
852	n/a	case FFI_TYPE_STRUCT:
853	n/a	if (is_hfa (cif->rtype))
854	n/a	{
855	n/a	int j;
856	n/a	unsigned short type = get_homogeneous_type (cif->rtype);
857	n/a	unsigned elems = element_count (cif->rtype);
858	n/a	for (j = 0; j < elems; j++)
859	n/a	{
860	n/a	void *reg = get_basic_type_addr (type, &context, j);
861	n/a	copy_basic_type (rvalue, reg, type);
862	n/a	rvalue += get_basic_type_size (type);
863	n/a	}
864	n/a	}
865	n/a	else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
866	n/a	{
867	n/a	size_t size = ALIGN (cif->rtype->size, sizeof (UINT64));
868	n/a	memcpy (rvalue, get_x_addr (&context, 0), size);
869	n/a	}
870	n/a	else
871	n/a	{
872	n/a	FFI_ASSERT (0);
873	n/a	}
874	n/a	break;
875	n/a
876	n/a	default:
877	n/a	FFI_ASSERT (0);
878	n/a	break;
879	n/a	}
880	n/a	}
881	n/a	else
882	n/a	{
883	n/a	memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64));
884	n/a	ffi_call_SYSV (aarch64_prep_args, &context, &ecif,
885	n/a	stack_bytes, fn);
886	n/a	}
887	n/a	break;
888	n/a	}
889	n/a
890	n/a	default:
891	n/a	FFI_ASSERT (0);
892	n/a	break;
893	n/a	}
894	n/a	}
895	n/a
896	n/a	static unsigned char trampoline [] =
897	n/a	{ 0x70, 0x00, 0x00, 0x58, /* ldr x16, 1f */
898	n/a	0x91, 0x00, 0x00, 0x10, /* adr x17, 2f */
899	n/a	0x00, 0x02, 0x1f, 0xd6 /* br x16 */
900	n/a	};
901	n/a
902	n/a	/* Build a trampoline. */
903	n/a
904	n/a	#define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \
905	n/a	({unsigned char __tramp = (unsigned char)(TRAMP); \
906	n/a	UINT64 __fun = (UINT64)(FUN); \
907	n/a	UINT64 __ctx = (UINT64)(CTX); \
908	n/a	UINT64 __flags = (UINT64)(FLAGS); \
909	n/a	memcpy (__tramp, trampoline, sizeof (trampoline)); \
910	n/a	memcpy (__tramp + 12, &__fun, sizeof (__fun)); \
911	n/a	memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \
912	n/a	memcpy (__tramp + 28, &__flags, sizeof (__flags)); \
913	n/a	ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \
914	n/a	})
915	n/a
916	n/a	ffi_status
917	n/a	ffi_prep_closure_loc (ffi_closure* closure,
918	n/a	ffi_cif* cif,
919	n/a	void (fun)(ffi_cif,void,void,void),
920	n/a	void *user_data,
921	n/a	void *codeloc)
922	n/a	{
923	n/a	if (cif->abi != FFI_SYSV)
924	n/a	return FFI_BAD_ABI;
925	n/a
926	n/a	FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc,
927	n/a	cif->aarch64_flags);
928	n/a
929	n/a	closure->cif = cif;
930	n/a	closure->user_data = user_data;
931	n/a	closure->fun = fun;
932	n/a
933	n/a	return FFI_OK;
934	n/a	}
935	n/a
936	n/a	/* Primary handler to setup and invoke a function within a closure.
937	n/a
938	n/a	A closure when invoked enters via the assembler wrapper
939	n/a	ffi_closure_SYSV(). The wrapper allocates a call context on the
940	n/a	stack, saves the interesting registers (from the perspective of
941	n/a	the calling convention) into the context then passes control to
942	n/a	ffi_closure_SYSV_inner() passing the saved context and a pointer to
943	n/a	the stack at the point ffi_closure_SYSV() was invoked.
944	n/a
945	n/a	On the return path the assembler wrapper will reload call context
946	n/a	registers.
947	n/a
948	n/a	ffi_closure_SYSV_inner() marshalls the call context into ffi value
949	n/a	descriptors, invokes the wrapped function, then marshalls the return
950	n/a	value back into the call context. */
951	n/a
952	n/a	void FFI_HIDDEN
953	n/a	ffi_closure_SYSV_inner (ffi_closure closure, struct call_context context,
954	n/a	void *stack)
955	n/a	{
956	n/a	ffi_cif *cif = closure->cif;
957	n/a	void avalue = (void) alloca (cif->nargs * sizeof (void*));
958	n/a	void *rvalue = NULL;
959	n/a	int i;
960	n/a	struct arg_state state;
961	n/a
962	n/a	arg_init (&state, ALIGN(cif->bytes, 16));
963	n/a
964	n/a	for (i = 0; i < cif->nargs; i++)
965	n/a	{
966	n/a	ffi_type *ty = cif->arg_types[i];
967	n/a
968	n/a	switch (ty->type)
969	n/a	{
970	n/a	case FFI_TYPE_VOID:
971	n/a	FFI_ASSERT (0);
972	n/a	break;
973	n/a
974	n/a	case FFI_TYPE_UINT8:
975	n/a	case FFI_TYPE_SINT8:
976	n/a	case FFI_TYPE_UINT16:
977	n/a	case FFI_TYPE_SINT16:
978	n/a	case FFI_TYPE_UINT32:
979	n/a	case FFI_TYPE_SINT32:
980	n/a	case FFI_TYPE_INT:
981	n/a	case FFI_TYPE_POINTER:
982	n/a	case FFI_TYPE_UINT64:
983	n/a	case FFI_TYPE_SINT64:
984	n/a	case FFI_TYPE_FLOAT:
985	n/a	case FFI_TYPE_DOUBLE:
986	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
987	n/a	case FFI_TYPE_LONGDOUBLE:
988	n/a	avalue[i] = allocate_to_register_or_stack (context, stack,
989	n/a	&state, ty->type);
990	n/a	break;
991	n/a	#endif
992	n/a
993	n/a	case FFI_TYPE_STRUCT:
994	n/a	if (is_hfa (ty))
995	n/a	{
996	n/a	unsigned n = element_count (ty);
997	n/a	if (available_v (&state) < n)
998	n/a	{
999	n/a	state.nsrn = N_V_ARG_REG;
1000	n/a	avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
1001	n/a	ty->size);
1002	n/a	}
1003	n/a	else
1004	n/a	{
1005	n/a	switch (get_homogeneous_type (ty))
1006	n/a	{
1007	n/a	case FFI_TYPE_FLOAT:
1008	n/a	{
1009	n/a	/* Eeek! We need a pointer to the structure,
1010	n/a	however the homogeneous float elements are
1011	n/a	being passed in individual S registers,
1012	n/a	therefore the structure is not represented as
1013	n/a	a contiguous sequence of bytes in our saved
1014	n/a	register context. We need to fake up a copy
1015	n/a	of the structure laid out in memory
1016	n/a	correctly. The fake can be tossed once the
1017	n/a	closure function has returned hence alloca()
1018	n/a	is sufficient. */
1019	n/a	int j;
1020	n/a	UINT32 *p = avalue[i] = alloca (ty->size);
1021	n/a	for (j = 0; j < element_count (ty); j++)
1022	n/a	memcpy (&p[j],
1023	n/a	allocate_to_s (context, &state),
1024	n/a	sizeof (*p));
1025	n/a	break;
1026	n/a	}
1027	n/a
1028	n/a	case FFI_TYPE_DOUBLE:
1029	n/a	{
1030	n/a	/* Eeek! We need a pointer to the structure,
1031	n/a	however the homogeneous float elements are
1032	n/a	being passed in individual S registers,
1033	n/a	therefore the structure is not represented as
1034	n/a	a contiguous sequence of bytes in our saved
1035	n/a	register context. We need to fake up a copy
1036	n/a	of the structure laid out in memory
1037	n/a	correctly. The fake can be tossed once the
1038	n/a	closure function has returned hence alloca()
1039	n/a	is sufficient. */
1040	n/a	int j;
1041	n/a	UINT64 *p = avalue[i] = alloca (ty->size);
1042	n/a	for (j = 0; j < element_count (ty); j++)
1043	n/a	memcpy (&p[j],
1044	n/a	allocate_to_d (context, &state),
1045	n/a	sizeof (*p));
1046	n/a	break;
1047	n/a	}
1048	n/a
1049	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1050	n/a	case FFI_TYPE_LONGDOUBLE:
1051	n/a	memcpy (&avalue[i],
1052	n/a	allocate_to_v (context, &state),
1053	n/a	sizeof (*avalue));
1054	n/a	break;
1055	n/a	#endif
1056	n/a
1057	n/a	default:
1058	n/a	FFI_ASSERT (0);
1059	n/a	break;
1060	n/a	}
1061	n/a	}
1062	n/a	}
1063	n/a	else if (ty->size > 16)
1064	n/a	{
1065	n/a	/* Replace Composite type of size greater than 16 with a
1066	n/a	pointer. */
1067	n/a	memcpy (&avalue[i],
1068	n/a	allocate_to_register_or_stack (context, stack,
1069	n/a	&state, FFI_TYPE_POINTER),
1070	n/a	sizeof (avalue[i]));
1071	n/a	}
1072	n/a	else if (available_x (&state) >= (ty->size + 7) / 8)
1073	n/a	{
1074	n/a	avalue[i] = get_x_addr (context, state.ngrn);
1075	n/a	state.ngrn += (ty->size + 7) / 8;
1076	n/a	}
1077	n/a	else
1078	n/a	{
1079	n/a	state.ngrn = N_X_ARG_REG;
1080	n/a
1081	n/a	avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
1082	n/a	ty->size);
1083	n/a	}
1084	n/a	break;
1085	n/a
1086	n/a	default:
1087	n/a	FFI_ASSERT (0);
1088	n/a	break;
1089	n/a	}
1090	n/a	}
1091	n/a
1092	n/a	/* Figure out where the return value will be passed, either in
1093	n/a	registers or in a memory block allocated by the caller and passed
1094	n/a	in x8. */
1095	n/a
1096	n/a	if (is_register_candidate (cif->rtype))
1097	n/a	{
1098	n/a	/* Register candidates are always returned in registers. */
1099	n/a
1100	n/a	/* Allocate a scratchpad for the return value, we will let the
1101	n/a	callee scrible the result into the scratch pad then move the
1102	n/a	contents into the appropriate return value location for the
1103	n/a	call convention. */
1104	n/a	rvalue = alloca (cif->rtype->size);
1105	n/a	(closure->fun) (cif, rvalue, avalue, closure->user_data);
1106	n/a
1107	n/a	/* Copy the return value into the call context so that it is returned
1108	n/a	as expected to our caller. */
1109	n/a	switch (cif->rtype->type)
1110	n/a	{
1111	n/a	case FFI_TYPE_VOID:
1112	n/a	break;
1113	n/a
1114	n/a	case FFI_TYPE_UINT8:
1115	n/a	case FFI_TYPE_UINT16:
1116	n/a	case FFI_TYPE_UINT32:
1117	n/a	case FFI_TYPE_POINTER:
1118	n/a	case FFI_TYPE_UINT64:
1119	n/a	case FFI_TYPE_SINT8:
1120	n/a	case FFI_TYPE_SINT16:
1121	n/a	case FFI_TYPE_INT:
1122	n/a	case FFI_TYPE_SINT32:
1123	n/a	case FFI_TYPE_SINT64:
1124	n/a	case FFI_TYPE_FLOAT:
1125	n/a	case FFI_TYPE_DOUBLE:
1126	n/a	#if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1127	n/a	case FFI_TYPE_LONGDOUBLE:
1128	n/a	#endif
1129	n/a	{
1130	n/a	void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
1131	n/a	copy_basic_type (addr, rvalue, cif->rtype->type);
1132	n/a	break;
1133	n/a	}
1134	n/a	case FFI_TYPE_STRUCT:
1135	n/a	if (is_hfa (cif->rtype))
1136	n/a	{
1137	n/a	int j;
1138	n/a	unsigned short type = get_homogeneous_type (cif->rtype);
1139	n/a	unsigned elems = element_count (cif->rtype);
1140	n/a	for (j = 0; j < elems; j++)
1141	n/a	{
1142	n/a	void *reg = get_basic_type_addr (type, context, j);
1143	n/a	copy_basic_type (reg, rvalue, type);
1144	n/a	rvalue += get_basic_type_size (type);
1145	n/a	}
1146	n/a	}
1147	n/a	else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
1148	n/a	{
1149	n/a	size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)) ;
1150	n/a	memcpy (get_x_addr (context, 0), rvalue, size);
1151	n/a	}
1152	n/a	else
1153	n/a	{
1154	n/a	FFI_ASSERT (0);
1155	n/a	}
1156	n/a	break;
1157	n/a	default:
1158	n/a	FFI_ASSERT (0);
1159	n/a	break;
1160	n/a	}
1161	n/a	}
1162	n/a	else
1163	n/a	{
1164	n/a	memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64));
1165	n/a	(closure->fun) (cif, rvalue, avalue, closure->user_data);
1166	n/a	}
1167	n/a	}
1168	n/a