incdep.c@ 2850

Last change on this file since 2850 was 2850, checked in by bird, 8 years ago
kmk: optimizations.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 53.1 KB

Line
1	#ifdef CONFIG_WITH_INCLUDEDEP
2	/* $Id: incdep.c 2850 2016-08-30 16:06:31Z bird $ */
3	/** @file
4	* incdep - Simple dependency files.
5	*/
6
7	/*
8	* Copyright (c) 2007-2010 knut st. osmundsen <[email protected]>
9	*
10	* This file is part of kBuild.
11	*
12	* kBuild is free software; you can redistribute it and/or modify
13	* it under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; either version 3 of the License, or
15	* (at your option) any later version.
16	*
17	* kBuild is distributed in the hope that it will be useful,
18	* but WITHOUT ANY WARRANTY; without even the implied warranty of
19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20	* GNU General Public License for more details.
21	*
22	* You should have received a copy of the GNU General Public License
23	* along with kBuild. If not, see <http://www.gnu.org/licenses/>
24	*
25	*/
26
27	/*******************************************************************************
28	* Header Files *
29	*******************************************************************************/
30	#ifdef __OS2__
31	# define INCL_BASE
32	# define INCL_ERRORS
33	#endif
34
35	#include "make.h"
36
37	#if !defined(WINDOWS32) && !defined(__OS2__)
38	# define HAVE_PTHREAD
39	#endif
40
41	#include <assert.h>
42
43	#include <glob.h>
44
45	#include "dep.h"
46	#include "filedef.h"
47	#include "job.h"
48	#include "commands.h"
49	#include "variable.h"
50	#include "rule.h"
51	#include "debug.h"
52	#include "strcache2.h"
53
54	#ifdef HAVE_FCNTL_H
55	# include <fcntl.h>
56	#else
57	# include <sys/file.h>
58	#endif
59
60	#ifdef WINDOWS32
61	# include <io.h>
62	# include <process.h>
63	# include <Windows.h>
64	# define PARSE_IN_WORKER
65	#endif
66
67	#ifdef __OS2__
68	# include <os2.h>
69	# include <sys/fmutex.h>
70	#endif
71
72	#ifdef HAVE_PTHREAD
73	# include <pthread.h>
74	#endif
75
76	#ifdef __APPLE__
77	# include <malloc/malloc.h>
78	# define PARSE_IN_WORKER
79	#endif
80
81	#if defined(__gnu_linux__) \|\| defined(__linux__)
82	# define PARSE_IN_WORKER
83	#endif
84
85
86	/*******************************************************************************
87	* Structures and Typedefs *
88	*******************************************************************************/
89	struct incdep_variable_in_set
90	{
91	struct incdep_variable_in_set *next;
92	/* the parameters */
93	struct strcache2_entry name_entry; / dep strcache - WRONG */
94	const char value; / xmalloc'ed */
95	unsigned int value_length;
96	int duplicate_value; /* 0 */
97	enum variable_origin origin;
98	int recursive;
99	struct variable_set *set;
100	const struct floc flocp; / NILF */
101	};
102
103	struct incdep_variable_def
104	{
105	struct incdep_variable_def *next;
106	/* the parameters */
107	const struct floc flocp; / NILF */
108	struct strcache2_entry name_entry; / dep strcache - WRONG */
109	char value; / xmalloc'ed, free it */
110	unsigned int value_length;
111	enum variable_origin origin;
112	enum variable_flavor flavor;
113	int target_var;
114	};
115
116	struct incdep_recorded_file
117	{
118	struct incdep_recorded_file *next;
119
120	/* the parameters */
121	struct strcache2_entry filename_entry; / dep strcache; converted to a nameseq record. */
122	struct dep deps; / All the names are dep strcache entries. */
123	const struct floc flocp; / NILF */
124	};
125
126
127	/* per dep file structure. */
128	struct incdep
129	{
130	struct incdep *next;
131	char *file_base;
132	char *file_end;
133
134	int worker_tid;
135	#ifdef PARSE_IN_WORKER
136	unsigned int err_line_no;
137	const char *err_msg;
138
139	struct incdep_variable_in_set *recorded_variables_in_set_head;
140	struct incdep_variable_in_set *recorded_variables_in_set_tail;
141
142	struct incdep_variable_def *recorded_variable_defs_head;
143	struct incdep_variable_def *recorded_variable_defs_tail;
144
145	struct incdep_recorded_file *recorded_file_head;
146	struct incdep_recorded_file *recorded_file_tail;
147	#endif
148
149	char name[1];
150	};
151
152
153	/*******************************************************************************
154	* Global Variables *
155	*******************************************************************************/
156
157	/* mutex protecting the globals and an associated condition/event. */
158	#ifdef HAVE_PTHREAD
159	static pthread_mutex_t incdep_mtx;
160	static pthread_cond_t incdep_cond_todo;
161	static pthread_cond_t incdep_cond_done;
162
163	#elif defined (WINDOWS32)
164	static CRITICAL_SECTION incdep_mtx;
165	static HANDLE incdep_hev_todo;
166	static HANDLE incdep_hev_done;
167	static int volatile incdep_hev_todo_waiters;
168	static int volatile incdep_hev_done_waiters;
169
170	#elif defined (__OS2__)
171	static _fmutex incdep_mtx;
172	static HEV incdep_hev_todo;
173	static HEV incdep_hev_done;
174	static int volatile incdep_hev_todo_waiters;
175	static int volatile incdep_hev_done_waiters;
176	#endif
177
178	/* flag indicating whether the threads, lock and event/condvars has
179	been initialized or not. */
180	static int incdep_initialized;
181
182	/* the list of files that needs reading. */
183	static struct incdep * volatile incdep_head_todo;
184	static struct incdep * volatile incdep_tail_todo;
185
186	/* the number of files that are currently being read. */
187	static int volatile incdep_num_reading;
188
189	/* the list of files that have been read. */
190	static struct incdep * volatile incdep_head_done;
191	static struct incdep * volatile incdep_tail_done;
192
193
194	/* The handles to the worker threads. */
195	#ifdef HAVE_PTHREAD
196	# define INCDEP_MAX_THREADS 1
197	static pthread_t incdep_threads[INCDEP_MAX_THREADS];
198
199	#elif defined (WINDOWS32)
200	# define INCDEP_MAX_THREADS 2
201	static HANDLE incdep_threads[INCDEP_MAX_THREADS];
202
203	#elif defined (__OS2__)
204	# define INCDEP_MAX_THREADS 2
205	static TID incdep_threads[INCDEP_MAX_THREADS];
206	#endif
207
208	static struct alloccache incdep_rec_caches[INCDEP_MAX_THREADS];
209	static struct alloccache incdep_dep_caches[INCDEP_MAX_THREADS];
210	static struct strcache2 incdep_dep_strcaches[INCDEP_MAX_THREADS];
211	static struct strcache2 incdep_var_strcaches[INCDEP_MAX_THREADS];
212	static unsigned incdep_num_threads;
213
214	/* flag indicating whether the worker threads should terminate or not. */
215	static int volatile incdep_terminate;
216
217	#ifdef __APPLE__
218	/* malloc zone for the incdep threads. */
219	static malloc_zone_t *incdep_zone;
220	#endif
221
222
223	/*******************************************************************************
224	* Internal Functions *
225	*******************************************************************************/
226	static void incdep_flush_it (struct floc *);
227	static void eval_include_dep_file (struct incdep , struct floc );
228	static void incdep_commit_recorded_file (const char filename, struct dep deps,
229	const struct floc *flocp);
230
231
232	/* xmalloc wrapper.
233	For working around multithreaded performance problems found on Darwin,
234	Linux (glibc), and possibly other systems. */
235	static void *
236	incdep_xmalloc (struct incdep *cur, size_t size)
237	{
238	void *ptr;
239
240	#ifdef __APPLE__
241	if (cur && cur->worker_tid != -1)
242	{
243	ptr = malloc_zone_malloc (incdep_zone, size);
244	if (!ptr)
245	fatal (NILF, _("virtual memory exhausted"));
246	}
247	else
248	ptr = xmalloc (size);
249	#else
250	ptr = xmalloc (size);
251	#endif
252
253	(void)cur;
254	return ptr;
255	}
256
257	#if 0
258	/* cmalloc wrapper */
259	static void *
260	incdep_xcalloc (struct incdep *cur, size_t size)
261	{
262	void *ptr;
263
264	#ifdef __APPLE__
265	if (cur && cur->worker_tid != -1)
266	ptr = malloc_zone_calloc (incdep_zone, size, 1);
267	else
268	ptr = calloc (size, 1);
269	#else
270	ptr = calloc (size, 1);
271	#endif
272	if (!ptr)
273	fatal (NILF, _("virtual memory exhausted"));
274
275	(void)cur;
276	return ptr;
277	}
278	#endif /* unused */
279
280	/* free wrapper */
281	static void
282	incdep_xfree (struct incdep cur, void ptr)
283	{
284	/* free() must work for the allocation hacks above because
285	of free_dep_chain. */
286	free (ptr);
287	(void)cur;
288	}
289
290	/* alloc a dep structure. These are allocated in bunches to save time. */
291	struct dep *
292	incdep_alloc_dep (struct incdep *cur)
293	{
294	struct alloccache *cache;
295	if (cur->worker_tid != -1)
296	cache = &incdep_dep_caches[cur->worker_tid];
297	else
298	cache = &dep_cache;
299	return alloccache_calloc (cache);
300	}
301
302	/* duplicates the dependency list pointed to by srcdep. */
303	static struct dep *
304	incdep_dup_dep_list (struct incdep cur, struct dep const srcdep)
305	{
306	struct alloccache *cache;
307	struct dep *retdep;
308	struct dep *dstdep;
309
310	if (cur->worker_tid != -1)
311	cache = &incdep_dep_caches[cur->worker_tid];
312	else
313	cache = &dep_cache;
314
315	if (srcdep)
316	{
317	retdep = dstdep = alloccache_alloc (cache);
318	for (;;)
319	{
320	dstdep->name = srcdep->name; /* string cached */
321	dstdep->includedep = srcdep->includedep;
322	srcdep = srcdep->next;
323	if (!srcdep)
324	{
325	dstdep->next = NULL;
326	break;
327	}
328	dstdep->next = alloccache_alloc (cache);
329	dstdep = dstdep->next;
330	}
331	}
332	else
333	retdep = NULL;
334	return retdep;
335	}
336
337
338	/* allocate a record. */
339	static void *
340	incdep_alloc_rec (struct incdep *cur)
341	{
342	return alloccache_alloc (&incdep_rec_caches[cur->worker_tid]);
343	}
344
345	/* free a record. */
346	static void
347	incdep_free_rec (struct incdep cur, void rec)
348	{
349	/alloccache_free (&incdep_rec_caches[cur->worker_tid], rec); - doesn't work of course. /
350	}
351
352
353	/* grow a cache. */
354	static void *
355	incdep_cache_allocator (void *thrd, unsigned int size)
356	{
357	(void)thrd;
358	#ifdef __APPLE__
359	return malloc_zone_malloc (incdep_zone, size);
360	#else
361	return xmalloc (size);
362	#endif
363	}
364
365	/* term a cache. */
366	static void
367	incdep_cache_deallocator (void thrd, void ptr, unsigned int size)
368	{
369	(void)thrd;
370	(void)size;
371	free (ptr);
372	}
373
374	/* acquires the lock */
375	void
376	incdep_lock(void)
377	{
378	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
379	pthread_mutex_lock (&incdep_mtx);
380	#elif defined (WINDOWS32)
381	EnterCriticalSection (&incdep_mtx);
382	#elif defined (__OS2__)
383	_fmutex_request (&incdep_mtx, 0);
384	#endif
385	}
386
387	/* releases the lock */
388	void
389	incdep_unlock(void)
390	{
391	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
392	pthread_mutex_unlock (&incdep_mtx);
393	#elif defined(WINDOWS32)
394	LeaveCriticalSection (&incdep_mtx);
395	#elif defined(__OS2__)
396	_fmutex_release (&incdep_mtx);
397	#endif
398	}
399
400	/* signals the main thread that there is stuff todo. caller owns the lock. */
401	static void
402	incdep_signal_done (void)
403	{
404	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
405	pthread_cond_broadcast (&incdep_cond_done);
406	#elif defined (WINDOWS32)
407	if (incdep_hev_done_waiters)
408	SetEvent (incdep_hev_done);
409	#elif defined (__OS2__)
410	if (incdep_hev_done_waiters)
411	DosPostEventSem (incdep_hev_done);
412	#endif
413	}
414
415	/* waits for a reader to finish reading. caller owns the lock. */
416	static void
417	incdep_wait_done (void)
418	{
419	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
420	pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
421
422	#elif defined (WINDOWS32)
423	ResetEvent (incdep_hev_done);
424	incdep_hev_done_waiters++;
425	incdep_unlock ();
426	WaitForSingleObject (incdep_hev_done, INFINITE);
427	incdep_lock ();
428	incdep_hev_done_waiters--;
429
430	#elif defined (__OS2__)
431	ULONG ulIgnore;
432	DosResetEventSem (incdep_hev_done, &ulIgnore);
433	incdep_hev_done_waiters++;
434	incdep_unlock ();
435	DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
436	incdep_lock ();
437	incdep_hev_done_waiters--;
438	#endif
439	}
440
441	/* signals the worker threads. caller owns the lock. */
442	static void
443	incdep_signal_todo (void)
444	{
445	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
446	pthread_cond_broadcast (&incdep_cond_todo);
447	#elif defined(WINDOWS32)
448	if (incdep_hev_todo_waiters)
449	SetEvent (incdep_hev_todo);
450	#elif defined(__OS2__)
451	if (incdep_hev_todo_waiters)
452	DosPostEventSem (incdep_hev_todo);
453	#endif
454	}
455
456	/* waits for stuff to arrive in the todo list. caller owns the lock. */
457	static void
458	incdep_wait_todo (void)
459	{
460	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
461	pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
462
463	#elif defined (WINDOWS32)
464	ResetEvent (incdep_hev_todo);
465	incdep_hev_todo_waiters++;
466	incdep_unlock ();
467	WaitForSingleObject (incdep_hev_todo, INFINITE);
468	incdep_lock ();
469	incdep_hev_todo_waiters--;
470
471	#elif defined (__OS2__)
472	ULONG ulIgnore;
473	DosResetEventSem (incdep_hev_todo, &ulIgnore);
474	incdep_hev_todo_waiters++;
475	incdep_unlock ();
476	DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
477	incdep_lock ();
478	incdep_hev_todo_waiters--;
479	#endif
480	}
481
482	/* Reads a dep file into memory. */
483	static int
484	incdep_read_file (struct incdep cur, struct floc f)
485	{
486	int fd;
487	struct stat st;
488
489	errno = 0;
490	#ifdef O_BINARY
491	fd = open (cur->name, O_RDONLY \| O_BINARY, 0);
492	#else
493	fd = open (cur->name, O_RDONLY, 0);
494	#endif
495	if (fd < 0)
496	{
497	/* ignore non-existing dependency files. */
498	int err = errno;
499	if (err == ENOENT \|\| stat (cur->name, &st) != 0)
500	return 1;
501	error (f, "%s: %s", cur->name, strerror (err));
502	return -1;
503	}
504	#ifdef KBUILD_OS_WINDOWS /* fewer kernel calls */
505	if (!birdStatOnFdJustSize (fd, &st.st_size))
506	#else
507	if (!fstat (fd, &st))
508	#endif
509	{
510	cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
511	if (read (fd, cur->file_base, st.st_size) == st.st_size)
512	{
513	close (fd);
514	cur->file_end = cur->file_base + st.st_size;
515	cur->file_base[st.st_size] = '\0';
516	return 0;
517	}
518
519	/* bail out */
520
521	error (f, "%s: read: %s", cur->name, strerror (errno));
522	incdep_xfree (cur, cur->file_base);
523	}
524	else
525	error (f, "%s: fstat: %s", cur->name, strerror (errno));
526
527	close (fd);
528	cur->file_base = cur->file_end = NULL;
529	return -1;
530	}
531
532	/* Free the incdep structure. */
533	static void
534	incdep_freeit (struct incdep *cur)
535	{
536	#ifdef PARSE_IN_WORKER
537	assert (!cur->recorded_variables_in_set_head);
538	assert (!cur->recorded_variable_defs_head);
539	assert (!cur->recorded_file_head);
540	#endif
541
542	incdep_xfree (cur, cur->file_base);
543	cur->next = NULL;
544	free (cur);
545	}
546
547	/* A worker thread. */
548	void
549	incdep_worker (int thrd)
550	{
551	incdep_lock ();
552
553	while (!incdep_terminate)
554	{
555	/* get job from the todo list. */
556
557	struct incdep *cur = incdep_head_todo;
558	if (!cur)
559	{
560	incdep_wait_todo ();
561	continue;
562	}
563	if (cur->next)
564	incdep_head_todo = cur->next;
565	else
566	incdep_head_todo = incdep_tail_todo = NULL;
567	incdep_num_reading++;
568
569	/* read the file. */
570
571	incdep_unlock ();
572	cur->worker_tid = thrd;
573
574	incdep_read_file (cur, NILF);
575	#ifdef PARSE_IN_WORKER
576	eval_include_dep_file (cur, NILF);
577	#endif
578
579	cur->worker_tid = -1;
580	incdep_lock ();
581
582	/* insert finished job into the done list. */
583
584	incdep_num_reading--;
585	cur->next = NULL;
586	if (incdep_tail_done)
587	incdep_tail_done->next = cur;
588	else
589	incdep_head_done = cur;
590	incdep_tail_done = cur;
591
592	incdep_signal_done ();
593	}
594
595	incdep_unlock ();
596	}
597
598	/* Thread library specific thread functions wrapping incdep_wroker. */
599	#ifdef HAVE_PTHREAD
600	static void *
601	incdep_worker_pthread (void *thrd)
602	{
603	incdep_worker ((size_t)thrd);
604	return NULL;
605	}
606
607	#elif defined (WINDOWS32)
608	static unsigned __stdcall
609	incdep_worker_windows (void *thrd)
610	{
611	incdep_worker ((size_t)thrd);
612	return 0;
613	}
614
615	#elif defined (__OS2__)
616	static void
617	incdep_worker_os2 (void *thrd)
618	{
619	incdep_worker ((size_t)thrd);
620	}
621	#endif
622
623	/* Checks if threads are enabled or not.
624
625	This is a special hack so that is possible to disable the threads when in a
626	debian fakeroot environment. Thus, in addition to the KMK_THREADS_DISABLED
627	and KMK_THREADS_ENABLED environment variable check we also check for signs
628	of fakeroot. */
629	static int
630	incdep_are_threads_enabled (void)
631	{
632	#if defined (CONFIG_WITHOUT_THREADS)
633	return 0;
634	#endif
635
636	/* Generic overrides. */
637	if (getenv ("KMK_THREADS_DISABLED"))
638	{
639	message (1, "Threads disabled (environment)");
640	return 0;
641	}
642	if (getenv ("KMK_THREADS_ENABLED"))
643	return 1;
644
645	#if defined (__gnu_linux__) \|\| defined (__linux__)
646	/* Try detect fakeroot. */
647	if (getenv ("FAKEROOTKEY")
648	\|\| getenv ("FAKEROOTUID")
649	\|\| getenv ("FAKEROOTGID")
650	\|\| getenv ("FAKEROOTEUID")
651	\|\| getenv ("FAKEROOTEGID")
652	\|\| getenv ("FAKEROOTSUID")
653	\|\| getenv ("FAKEROOTSGID")
654	\|\| getenv ("FAKEROOTFUID")
655	\|\| getenv ("FAKEROOTFGID")
656	\|\| getenv ("FAKEROOTDONTTRYCHOWN")
657	\|\| getenv ("FAKEROOT_FD_BASE")
658	\|\| getenv ("FAKEROOT_DB_SEARCH_PATHS"))
659	{
660	message (1, "Threads disabled (fakeroot)");
661	return 0;
662	}
663
664	/* LD_PRELOAD could indicate undetected debian fakeroot or some
665	other ingenius library which cannot deal correctly with threads. */
666	if (getenv ("LD_PRELOAD"))
667	{
668	message (1, "Threads disabled (LD_PRELOAD)");
669	return 0;
670	}
671
672	#elif defined(__APPLE__) \
673	\|\| defined(__sun__) \|\| defined(__SunOS__) \|\| defined(__sun) \|\| defined(__SunOS) \
674	\|\| defined(__FreeBSD__) \|\| defined(__OpenBSD__) \|\| defined(__NetBSD__) \|\| defined(__DragonFly__) \
675	\|\| defined(__HAIKU__)
676	/* No broken preload libraries known to be in common use on these platforms... */
677
678	#elif defined(_MSC_VER) \|\| defined(_WIN32) \|\| defined(__OS2__)
679	/* No preload mess to care about. */
680
681	#else
682	# error "Add your self to the appropriate case above and send a patch to bird."
683	#endif
684	return 1;
685	}
686
687	/* Creates the the worker threads. */
688	static void
689	incdep_init (struct floc *f)
690	{
691	unsigned i;
692	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
693	int rc;
694	pthread_attr_t attr;
695
696	#elif defined (WINDOWS32)
697	unsigned tid;
698	uintptr_t hThread;
699
700	#elif defined (__OS2__)
701	int rc;
702	int tid;
703	#endif
704	(void)f;
705
706	/* heap hacks */
707
708	#ifdef __APPLE__
709	incdep_zone = malloc_create_zone (0, 0);
710	if (!incdep_zone)
711	incdep_zone = malloc_default_zone ();
712	#endif
713
714
715	/* create the mutex and two condition variables / event objects. */
716
717	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
718	rc = pthread_mutex_init (&incdep_mtx, NULL);
719	if (rc)
720	fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
721	rc = pthread_cond_init (&incdep_cond_todo, NULL);
722	if (rc)
723	fatal (f, _("pthread_cond_init failed: err=%d"), rc);
724	rc = pthread_cond_init (&incdep_cond_done, NULL);
725	if (rc)
726	fatal (f, _("pthread_cond_init failed: err=%d"), rc);
727
728	#elif defined (WINDOWS32)
729	InitializeCriticalSection (&incdep_mtx);
730	incdep_hev_todo = CreateEvent (NULL, TRUE /bManualReset/, FALSE /bInitialState/, NULL);
731	if (!incdep_hev_todo)
732	fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
733	incdep_hev_done = CreateEvent (NULL, TRUE /bManualReset/, FALSE /bInitialState/, NULL);
734	if (!incdep_hev_done)
735	fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
736	incdep_hev_todo_waiters = 0;
737	incdep_hev_done_waiters = 0;
738
739	#elif defined (__OS2__)
740	_fmutex_create (&incdep_mtx, 0);
741	rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
742	if (rc)
743	fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
744	rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
745	if (rc)
746	fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
747	incdep_hev_todo_waiters = 0;
748	incdep_hev_done_waiters = 0;
749	#endif
750
751	/* create the worker threads and associated per thread data. */
752
753	incdep_terminate = 0;
754	if (incdep_are_threads_enabled())
755	{
756	incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
757	if (incdep_num_threads + 1 > job_slots)
758	incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
759	for (i = 0; i < incdep_num_threads; i++)
760	{
761	/* init caches */
762	unsigned rec_size = sizeof (struct incdep_variable_in_set);
763	if (rec_size < sizeof (struct incdep_variable_def))
764	rec_size = sizeof (struct incdep_variable_def);
765	if (rec_size < sizeof (struct incdep_recorded_file))
766	rec_size = sizeof (struct incdep_recorded_file);
767	alloccache_init (&incdep_rec_caches[i], rec_size, "incdep rec",
768	incdep_cache_allocator, (void *)(size_t)i);
769	alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
770	incdep_cache_allocator, (void *)(size_t)i);
771	strcache2_init (&incdep_dep_strcaches[i],
772	"incdep dep", /* name */
773	65536, /* hash size */
774	0, /* default segment size*/
775	#ifdef HAVE_CASE_INSENSITIVE_FS
776	1, /* case insensitive */
777	#else
778	0, /* case insensitive */
779	#endif
780	0); /* thread safe */
781
782	strcache2_init (&incdep_var_strcaches[i],
783	"incdep var", /* name */
784	32768, /* hash size */
785	0, /* default segment size*/
786	0, /* case insensitive */
787	0); /* thread safe */
788
789	/* create the thread. */
790	#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
791	rc = pthread_attr_init (&attr);
792	if (rc)
793	fatal (f, _("pthread_attr_init failed: err=%d"), rc);
794	/rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); /
795	rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
796	if (rc)
797	fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
798	rc = pthread_create (&incdep_threads[i], &attr,
799	incdep_worker_pthread, (void *)(size_t)i);
800	if (rc)
801	fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
802	pthread_attr_destroy (&attr);
803
804	#elif defined (WINDOWS32)
805	tid = 0;
806	hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
807	(void *)i, 0, &tid);
808	if (hThread == 0 \|\| hThread == ~(uintptr_t)0)
809	fatal (f, _("_beginthreadex failed: err=%d"), errno);
810	incdep_threads[i] = (HANDLE)hThread;
811
812	#elif defined (__OS2__)
813	tid = _beginthread (incdep_worker_os2, NULL, 1281024, (void )i);
814	if (tid <= 0)
815	fatal (f, _("_beginthread failed: err=%d"), errno);
816	incdep_threads[i] = tid;
817	#endif
818	}
819	}
820	else
821	incdep_num_threads = 0;
822
823	incdep_initialized = 1;
824	}
825
826	/* Flushes outstanding work and terminates the worker threads.
827	This is called from snap_deps(). */
828	void
829	incdep_flush_and_term (void)
830	{
831	unsigned i;
832
833	if (!incdep_initialized)
834	return;
835
836	/* flush any out standing work */
837
838	incdep_flush_it (NILF);
839
840	/* tell the threads to terminate */
841
842	incdep_lock ();
843	incdep_terminate = 1;
844	incdep_signal_todo ();
845	incdep_unlock ();
846
847	/* wait for the threads to quit */
848
849	for (i = 0; i < incdep_num_threads; i++)
850	{
851	/* more later? */
852
853	/* terminate or join up the allocation caches. */
854	alloccache_term (&incdep_rec_caches[i], incdep_cache_deallocator, (void *)(size_t)i);
855	alloccache_join (&dep_cache, &incdep_dep_caches[i]);
856	strcache2_term (&incdep_dep_strcaches[i]);
857	strcache2_term (&incdep_var_strcaches[i]);
858	}
859	incdep_num_threads = 0;
860
861	/* destroy the lock and condition variables / event objects. */
862
863	/* later */
864
865	incdep_initialized = 0;
866	}
867
868	#ifdef PARSE_IN_WORKER
869	/* Flushes a strcache entry returning the actual string cache entry.
870	The input is freed! */
871	static const char *
872	incdep_flush_strcache_entry (struct strcache2_entry *entry)
873	{
874	if (!entry->user)
875	entry->user = (void *) strcache2_add_hashed_file (&file_strcache,
876	(const char *)(entry + 1),
877	entry->length, entry->hash);
878	return (const char *)entry->user;
879	}
880
881	/* Flushes the recorded instructions. */
882	static void
883	incdep_flush_recorded_instructions (struct incdep *cur)
884	{
885	struct incdep_variable_in_set *rec_vis;
886	struct incdep_variable_def *rec_vd;
887	struct incdep_recorded_file *rec_f;
888
889	/* Display saved error. */
890
891	if (cur->err_msg)
892	error(NILF, "%s(%d): %s", cur->name, cur->err_line_no, cur->err_msg);
893
894
895	/* define_variable_in_set */
896
897	rec_vis = cur->recorded_variables_in_set_head;
898	cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
899	if (rec_vis)
900	do
901	{
902	void *free_me = rec_vis;
903	unsigned int name_length = rec_vis->name_entry->length;
904	define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
905	name_length,
906	rec_vis->value,
907	rec_vis->value_length,
908	rec_vis->duplicate_value,
909	rec_vis->origin,
910	rec_vis->recursive,
911	rec_vis->set,
912	rec_vis->flocp);
913	rec_vis = rec_vis->next;
914	incdep_free_rec (cur, free_me);
915	}
916	while (rec_vis);
917
918	/* do_variable_definition */
919
920	rec_vd = cur->recorded_variable_defs_head;
921	cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
922	if (rec_vd)
923	do
924	{
925	void *free_me = rec_vd;
926	do_variable_definition_2 (rec_vd->flocp,
927	incdep_flush_strcache_entry (rec_vd->name_entry),
928	rec_vd->value,
929	rec_vd->value_length,
930	0,
931	rec_vd->value,
932	rec_vd->origin,
933	rec_vd->flavor,
934	rec_vd->target_var);
935	rec_vd = rec_vd->next;
936	incdep_free_rec (cur, free_me);
937	}
938	while (rec_vd);
939
940	/* record_files */
941
942	rec_f = cur->recorded_file_head;
943	cur->recorded_file_head = cur->recorded_file_tail = NULL;
944	if (rec_f)
945	do
946	{
947	void *free_me = rec_f;
948	struct dep *dep;
949
950	for (dep = rec_f->deps; dep; dep = dep->next)
951	dep->name = incdep_flush_strcache_entry ((struct strcache2_entry *)dep->name);
952
953	incdep_commit_recorded_file (incdep_flush_strcache_entry (rec_f->filename_entry),
954	rec_f->deps,
955	rec_f->flocp);
956
957	rec_f = rec_f->next;
958	incdep_free_rec (cur, free_me);
959	}
960	while (rec_f);
961	}
962	#endif /* PARSE_IN_WORKER */
963
964	/* Record / issue a warning about a misformed dep file. */
965	static void
966	incdep_warn (struct incdep cur, unsigned int line_no, const char msg)
967	{
968	if (cur->worker_tid == -1)
969	error (NILF, "%s(%d): %s", cur->name, line_no, msg);
970	#ifdef PARSE_IN_WORKER
971	else
972	{
973	cur->err_line_no = line_no;
974	cur->err_msg = msg;
975	}
976	#endif
977	}
978
979	/* Dependency or file strcache allocation / recording. */
980	static const char *
981	incdep_dep_strcache (struct incdep cur, const char str, int len)
982	{
983	const char *ret;
984	if (cur->worker_tid == -1)
985	{
986	/* Make sure the string is terminated before we hand it to
987	strcache_add_len so it does have to make a temporary copy
988	of it on the stack. */
989	char ch = str[len];
990	((char *)str)[len] = '\0';
991	ret = strcache_add_len (str, len);
992	((char *)str)[len] = ch;
993	}
994	else
995	{
996	/* Add it out the strcache of the thread. */
997	ret = strcache2_add (&incdep_dep_strcaches[cur->worker_tid], str, len);
998	ret = (const char *)strcache2_get_entry(&incdep_dep_strcaches[cur->worker_tid], ret);
999	}
1000	return ret;
1001	}
1002
1003	/* Variable name allocation / recording. */
1004	static const char *
1005	incdep_var_strcache (struct incdep cur, const char str, int len)
1006	{
1007	const char *ret;
1008	if (cur->worker_tid == -1)
1009	{
1010	/* XXX: we're leaking this memory now! This will be fixed later. */
1011	ret = xmalloc (len + 1);
1012	memcpy ((char *)ret, str, len);
1013	((char *)ret)[len] = '\0';
1014	}
1015	else
1016	{
1017	/* Add it out the strcache of the thread. */
1018	ret = strcache2_add (&incdep_var_strcaches[cur->worker_tid], str, len);
1019	ret = (const char *)strcache2_get_entry(&incdep_var_strcaches[cur->worker_tid], ret);
1020	}
1021	return ret;
1022	}
1023
1024	/* Record / perform a variable definition in a set.
1025	The NAME is in the string cache.
1026	The VALUE is on the heap.
1027	The DUPLICATE_VALUE is always 0. */
1028	static void
1029	incdep_record_variable_in_set (struct incdep *cur,
1030	const char *name, unsigned int name_length,
1031	const char *value,
1032	unsigned int value_length,
1033	int duplicate_value,
1034	enum variable_origin origin,
1035	int recursive,
1036	struct variable_set *set,
1037	const struct floc *flocp)
1038	{
1039	assert (!duplicate_value);
1040	if (cur->worker_tid == -1)
1041	define_variable_in_set (name, name_length, value, value_length,
1042	duplicate_value, origin, recursive, set, flocp);
1043	#ifdef PARSE_IN_WORKER
1044	else
1045	{
1046	struct incdep_variable_in_set *rec =
1047	(struct incdep_variable_in_set *)incdep_alloc_rec (cur);
1048	rec->name_entry = (struct strcache2_entry *)name;
1049	rec->value = value;
1050	rec->value_length = value_length;
1051	rec->duplicate_value = duplicate_value;
1052	rec->origin = origin;
1053	rec->recursive = recursive;
1054	rec->set = set;
1055	rec->flocp = flocp;
1056
1057	rec->next = NULL;
1058	if (cur->recorded_variables_in_set_tail)
1059	cur->recorded_variables_in_set_tail->next = rec;
1060	else
1061	cur->recorded_variables_in_set_head = rec;
1062	cur->recorded_variables_in_set_tail = rec;
1063	}
1064	#endif
1065	}
1066
1067	/* Record / perform a variable definition. The VALUE should be disposed of. */
1068	static void
1069	incdep_record_variable_def (struct incdep *cur,
1070	const struct floc *flocp,
1071	const char *name,
1072	unsigned int name_length,
1073	char *value,
1074	unsigned int value_length,
1075	enum variable_origin origin,
1076	enum variable_flavor flavor,
1077	int target_var)
1078	{
1079	if (cur->worker_tid == -1)
1080	do_variable_definition_2 (flocp, name, value, value_length, 0, value,
1081	origin, flavor, target_var);
1082	#ifdef PARSE_IN_WORKER
1083	else
1084	{
1085	struct incdep_variable_def *rec =
1086	(struct incdep_variable_def *)incdep_alloc_rec (cur);
1087	rec->flocp = flocp;
1088	rec->name_entry = (struct strcache2_entry *)name;
1089	rec->value = value;
1090	rec->value_length = value_length;
1091	rec->origin = origin;
1092	rec->flavor = flavor;
1093	rec->target_var = target_var;
1094
1095	rec->next = NULL;
1096	if (cur->recorded_variable_defs_tail)
1097	cur->recorded_variable_defs_tail->next = rec;
1098	else
1099	cur->recorded_variable_defs_head = rec;
1100	cur->recorded_variable_defs_tail = rec;
1101	}
1102	#else
1103	(void)name_length;
1104	#endif
1105	}
1106
1107	/* Similar to record_files in read.c, only much much simpler. */
1108	static void
1109	incdep_commit_recorded_file (const char filename, struct dep deps,
1110	const struct floc *flocp)
1111	{
1112	struct file *f;
1113
1114	/* Perform some validations. */
1115	if (filename[0] == '.'
1116	&& ( streq(filename, ".POSIX")
1117	\|\| streq(filename, ".EXPORT_ALL_VARIABLES")
1118	\|\| streq(filename, ".INTERMEDIATE")
1119	\|\| streq(filename, ".LOW_RESOLUTION_TIME")
1120	\|\| streq(filename, ".NOTPARALLEL")
1121	\|\| streq(filename, ".ONESHELL")
1122	\|\| streq(filename, ".PHONY")
1123	\|\| streq(filename, ".PRECIOUS")
1124	\|\| streq(filename, ".SECONDARY")
1125	\|\| streq(filename, ".SECONDTARGETEXPANSION")
1126	\|\| streq(filename, ".SILENT")
1127	\|\| streq(filename, ".SHELLFLAGS")
1128	\|\| streq(filename, ".SUFFIXES")
1129	)
1130	)
1131	{
1132	error (flocp, _("reserved filename '%s' used in dependency file, ignored"), filename);
1133	return;
1134	}
1135
1136	/* Lookup or create an entry in the database. */
1137	f = enter_file (filename);
1138	if (f->double_colon)
1139	{
1140	error (flocp, _("dependency file '%s' has a double colon entry already, ignoring"), filename);
1141	return;
1142	}
1143	f->is_target = 1;
1144
1145	/* Append dependencies. */
1146	deps = enter_prereqs (deps, NULL);
1147	if (deps)
1148	{
1149	struct dep *last = f->deps;
1150	if (!last)
1151	f->deps = deps;
1152	else
1153	{
1154	while (last->next)
1155	last = last->next;
1156	last->next = deps;
1157	}
1158	}
1159	}
1160
1161	/* Record a file.*/
1162	static void
1163	incdep_record_file (struct incdep *cur,
1164	const char *filename,
1165	struct dep *deps,
1166	const struct floc *flocp)
1167	{
1168	if (cur->worker_tid == -1)
1169	incdep_commit_recorded_file (filename, deps, flocp);
1170	#ifdef PARSE_IN_WORKER
1171	else
1172	{
1173	struct incdep_recorded_file *rec =
1174	(struct incdep_recorded_file *) incdep_alloc_rec (cur);
1175
1176	rec->filename_entry = (struct strcache2_entry *)filename;
1177	rec->deps = deps;
1178	rec->flocp = flocp;
1179
1180	rec->next = NULL;
1181	if (cur->recorded_file_tail)
1182	cur->recorded_file_tail->next = rec;
1183	else
1184	cur->recorded_file_head = rec;
1185	cur->recorded_file_tail = rec;
1186	}
1187	#endif
1188	}
1189
1190
1191	/* no nonsense dependency file including.
1192
1193	Because nobody wants bogus dependency files to break their incremental
1194	builds with hard to comprehend error messages, this function does not
1195	use the normal eval routine but does all the parsing itself. This isn't,
1196	as much work as it sounds, because the necessary feature set is very
1197	limited.
1198
1199	eval_include_dep_file groks:
1200
1201	define var
1202	endef
1203
1204	var [\|:\|?\|>]= value [\]
1205
1206	[\]
1207	file: [deps] [\]
1208
1209	*/
1210	static void
1211	eval_include_dep_file (struct incdep curdep, struct floc f)
1212	{
1213	unsigned line_no = 1;
1214	const char *file_end = curdep->file_end;
1215	const char *cur = curdep->file_base;
1216	const char *endp;
1217
1218	/* if no file data, just return immediately. */
1219	if (!cur)
1220	return;
1221
1222	/* now parse the file. */
1223	while (cur < file_end)
1224	{
1225	/* skip empty lines */
1226	while (cur < file_end && isspace ((unsigned char)cur) && cur != '\n')
1227	++cur;
1228	if (cur >= file_end)
1229	break;
1230	if (*cur == '#')
1231	{
1232	cur = memchr (cur, '\n', file_end - cur);
1233	if (!cur)
1234	break;
1235	}
1236	if (*cur == '\\')
1237	{
1238	unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1239	: (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1240	: (file_end - cur == 1) ? 1 : 0;
1241	if (eol_len)
1242	{
1243	cur += eol_len;
1244	line_no++;
1245	continue;
1246	}
1247	}
1248	if (*cur == '\n')
1249	{
1250	cur++;
1251	line_no++;
1252	continue;
1253	}
1254
1255	/* define var
1256	...
1257	endef */
1258	if (strneq (cur, "define ", 7))
1259	{
1260	const char *var;
1261	unsigned var_len;
1262	const char *value_start;
1263	const char *value_end;
1264	char *value;
1265	unsigned value_len;
1266	int found_endef = 0;
1267
1268	/* extract the variable name. */
1269	cur += 7;
1270	while (isblank ((unsigned char)*cur))
1271	++cur;
1272	value_start = endp = memchr (cur, '\n', file_end - cur);
1273	if (!endp)
1274	endp = cur;
1275	while (endp > cur && isspace ((unsigned char)endp[-1]))
1276	--endp;
1277	var_len = endp - cur;
1278	if (!var_len)
1279	{
1280	incdep_warn (curdep, line_no, "bogus define statement.");
1281	break;
1282	}
1283	var = incdep_var_strcache (curdep, cur, var_len);
1284
1285	/* find the end of the variable. */
1286	cur = value_end = value_start = value_start + 1;
1287	++line_no;
1288	while (cur < file_end)
1289	{
1290	/* check for endef, don't bother with skipping leading spaces. */
1291	if ( file_end - cur >= 5
1292	&& strneq (cur, "endef", 5))
1293	{
1294	endp = cur + 5;
1295	while (endp < file_end && isspace ((unsigned char)endp) && endp != '\n')
1296	endp++;
1297	if (endp >= file_end \|\| *endp == '\n')
1298	{
1299	found_endef = 1;
1300	cur = endp >= file_end ? file_end : endp + 1;
1301	break;
1302	}
1303	}
1304
1305	/* skip a line ahead. */
1306	cur = value_end = memchr (cur, '\n', file_end - cur);
1307	if (cur != NULL)
1308	++cur;
1309	else
1310	cur = value_end = file_end;
1311	++line_no;
1312	}
1313
1314	if (!found_endef)
1315	{
1316	incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1317	break;
1318	}
1319	value_len = value_end - value_start;
1320	if (memchr (value_start, '\0', value_len))
1321	{
1322	incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1323	break;
1324	}
1325
1326	/* make a copy of the value, converting \r\n to \n, and define it. */
1327	value = incdep_xmalloc (curdep, value_len + 1);
1328	endp = memchr (value_start, '\r', value_len);
1329	if (endp)
1330	{
1331	const char *src = value_start;
1332	char *dst = value;
1333	for (;;)
1334	{
1335	size_t len = endp - src;
1336	memcpy (dst, src, len);
1337	dst += len;
1338	src = endp;
1339	if (src + 1 < file_end && src[1] == '\n')
1340	src++; /* skip the '\r' */
1341	if (src >= value_end)
1342	break;
1343	endp = memchr (endp + 1, '\r', src - value_end);
1344	if (!endp)
1345	endp = value_end;
1346	}
1347	value_len = dst - value;
1348	}
1349	else
1350	memcpy (value, value_start, value_len);
1351	value [value_len] = '\0';
1352
1353	incdep_record_variable_in_set (curdep,
1354	var, var_len, value, value_len,
1355	0 /* don't duplicate */, o_file,
1356	0 /* defines are recursive but this is faster */,
1357	NULL /* global set */, f);
1358	}
1359
1360	/* file: deps
1361	OR
1362	variable [:]= value */
1363	else
1364	{
1365	const char *equalp;
1366	const char *eol;
1367
1368	/* Look for a colon or and equal sign. In the assignment case, we
1369	require it to be on the same line as the variable name to simplify
1370	the code. Because of clang, we cannot make the same assumptions
1371	with file dependencies. So, start with the equal. */
1372
1373	assert (*cur != '\n');
1374	eol = memchr (cur, '\n', file_end - cur);
1375	if (!eol)
1376	eol = file_end;
1377	equalp = memchr (cur, '=', eol - cur);
1378	if (equalp)
1379	{
1380	/* An assignment of some sort. */
1381	const char *var;
1382	unsigned var_len;
1383	const char *value_start;
1384	const char *value_end;
1385	char *value;
1386	unsigned value_len;
1387	unsigned multi_line = 0;
1388	enum variable_flavor flavor;
1389
1390	/* figure the flavor first. */
1391	flavor = f_recursive;
1392	if (equalp > cur)
1393	{
1394	if (equalp[-1] == ':')
1395	flavor = f_simple;
1396	else if (equalp[-1] == '?')
1397	flavor = f_conditional;
1398	else if (equalp[-1] == '+')
1399	flavor = f_append;
1400	else if (equalp[-1] == '>')
1401	flavor = f_prepend;
1402	}
1403
1404	/* extract the variable name. */
1405	endp = flavor == f_recursive ? equalp : equalp - 1;
1406	while (endp > cur && isblank ((unsigned char)endp[-1]))
1407	--endp;
1408	var_len = endp - cur;
1409	if (!var_len)
1410	{
1411	incdep_warn (curdep, line_no, "empty variable. (includedep)");
1412	break;
1413	}
1414	if ( memchr (cur, '$', var_len)
1415	\|\| memchr (cur, ' ', var_len)
1416	\|\| memchr (cur, '\t', var_len))
1417	{
1418	incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1419	break;
1420	}
1421	var = incdep_var_strcache (curdep, cur, var_len);
1422
1423	/* find the start of the value. */
1424	cur = equalp + 1;
1425	while (cur < file_end && isblank ((unsigned char)*cur))
1426	cur++;
1427	value_start = cur;
1428
1429	/* find the end of the value / line (this isn't 101% correct). */
1430	value_end = cur;
1431	while (cur < file_end)
1432	{
1433	endp = value_end = memchr (cur, '\n', file_end - cur);
1434	if (!value_end)
1435	value_end = file_end;
1436	if (value_end - 1 >= cur && value_end[-1] == '\r')
1437	--value_end;
1438	if (value_end - 1 < cur \|\| value_end[-1] != '\\')
1439	{
1440	cur = endp ? endp + 1 : file_end;
1441	break;
1442	}
1443	--value_end;
1444	if (value_end - 1 >= cur && value_end[-1] == '\\')
1445	{
1446	incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1447	cur = NULL;
1448	break;
1449	}
1450	if (!endp)
1451	{
1452	cur = file_end;
1453	break;
1454	}
1455
1456	cur = endp + 1;
1457	++multi_line;
1458	++line_no;
1459	}
1460	if (!cur)
1461	break;
1462	++line_no;
1463
1464	/* make a copy of the value, converting \r\n to \n, and define it. */
1465	value_len = value_end - value_start;
1466	value = incdep_xmalloc (curdep, value_len + 1);
1467	if (!multi_line)
1468	memcpy (value, value_start, value_len);
1469	else
1470	{
1471	/* unescape it */
1472	const char *src = value_start;
1473	char *dst = value;
1474	while (src < value_end)
1475	{
1476	const char *nextp;
1477
1478	endp = memchr (src, '\n', value_end - src);
1479	if (!endp)
1480	nextp = endp = value_end;
1481	else
1482	nextp = endp + 1;
1483	if (endp > src && endp[-1] == '\r')
1484	--endp;
1485	if (endp > src && endp[-1] == '\\')
1486	--endp;
1487
1488	if (src != value_start)
1489	*dst++ = ' ';
1490	memcpy (dst, src, endp - src);
1491	dst += endp - src;
1492	src = nextp;
1493	}
1494	value_len = dst - value;
1495	}
1496	value [value_len] = '\0';
1497
1498	/* do the definition */
1499	if (flavor == f_recursive
1500	\|\| ( flavor == f_simple
1501	&& !memchr (value, '$', value_len)))
1502	incdep_record_variable_in_set (curdep,
1503	var, var_len, value, value_len,
1504	0 /* don't duplicate */, o_file,
1505	flavor == f_recursive /* recursive */,
1506	NULL /* global set */, f);
1507	else
1508	incdep_record_variable_def (curdep,
1509	f, var, var_len, value, value_len,
1510	o_file, flavor, 0 /* not target var */);
1511	}
1512	else
1513	{
1514	/* Expecting: file: dependencies */
1515
1516	const char *filename;
1517	const char *fnnext;
1518	const char *fnend;
1519	const char *colonp;
1520	struct dep *deps = 0;
1521	struct dep **nextdep = &deps;
1522	struct dep *dep;
1523
1524
1525	/* Locate the next file colon. If it's not within the bounds of
1526	the current line, check that all new line chars are escaped,
1527	and simplify them while we're at it. */
1528
1529	colonp = memchr (cur, ':', file_end - cur);
1530	#ifdef HAVE_DOS_PATHS
1531	while ( colonp
1532	&& colonp + 1 < file_end
1533	&& (colonp[1] == '/' \|\| colonp[1] == '\\')
1534	&& colonp > cur
1535	&& isalpha ((unsigned char)colonp[-1])
1536	&& ( colonp == cur + 1
1537	\|\| strchr (" \t(", colonp[-2]) != 0))
1538	colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1539	#endif
1540	if (!colonp)
1541	{
1542	incdep_warn (curdep, line_no, "no colon.");
1543	break;
1544	}
1545	if ((uintptr_t)colonp >= (uintptr_t)eol)
1546	{
1547	const char *sol;
1548
1549	if (memchr (eol, '=', colonp - eol))
1550	{
1551	incdep_warn (curdep, line_no, "multi line assignment / dependency confusion.");
1552	break;
1553	}
1554
1555	sol = cur;
1556	do
1557	{
1558	char eol2 = (char )eol - 1;
1559	if ((uintptr_t)eol2 >= (uintptr_t)sol && eol2 == '\r') / DOS line endings. */
1560	eol2--;
1561	if ((uintptr_t)eol2 < (uintptr_t)sol \|\| *eol2 != '\\')
1562	incdep_warn (curdep, line_no, "no colon.");
1563	else if (eol2 != sol && eol2[-1] == '\\')
1564	incdep_warn (curdep, line_no, "fancy EOL escape. (includedep)");
1565	else
1566	{
1567	eol2[0] = ' ';
1568	eol2[1] = ' ';
1569	if (eol2 != eol - 1)
1570	eol2[2] = ' ';
1571	line_no++;
1572
1573	sol = eol + 1;
1574	eol = memchr (sol, '\n', colonp - sol);
1575	continue;
1576	}
1577	sol = NULL;
1578	break;
1579	}
1580	while (eol != NULL);
1581	if (!sol)
1582	break;
1583	}
1584
1585	/* Extract the first filename after trimming and basic checks. */
1586	fnend = colonp;
1587	while ((uintptr_t)fnend > (uintptr_t)cur && isblank ((unsigned char)fnend[-1]))
1588	--fnend;
1589	if (cur == fnend)
1590	{
1591	incdep_warn (curdep, line_no, "empty filename.");
1592	break;
1593	}
1594	if (memchr (cur, '$', fnend - cur))
1595	{
1596	incdep_warn (curdep, line_no, "fancy file name. (includedep)");
1597	break;
1598	}
1599
1600	fnnext = cur;
1601	while (fnnext != fnend && !isblank ((unsigned char)*fnnext))
1602	fnnext++;
1603	filename = incdep_dep_strcache (curdep, cur, fnnext - cur);
1604
1605	/* parse any dependencies. */
1606	cur = colonp + 1;
1607	while (cur < file_end)
1608	{
1609	/* skip blanks and count lines. */
1610	while (cur < file_end && isspace ((unsigned char)cur) && cur != '\n')
1611	++cur;
1612	if (cur >= file_end)
1613	break;
1614	if (*cur == '\n')
1615	{
1616	cur++;
1617	line_no++;
1618	break;
1619	}
1620
1621	/* continuation + eol? */
1622	if (*cur == '\\')
1623	{
1624	unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1625	: (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1626	: (file_end - cur == 1) ? 1 : 0;
1627	if (eol_len)
1628	{
1629	cur += eol_len;
1630	line_no++;
1631	continue;
1632	}
1633	}
1634
1635	/* find the end of the filename */
1636	endp = cur;
1637	while (endp < file_end && !isspace ((unsigned char)*endp))
1638	++endp;
1639
1640	/* add it to the list. */
1641	*nextdep = dep = incdep_alloc_dep (curdep);
1642	dep->name = incdep_dep_strcache (curdep, cur, endp - cur);
1643	dep->includedep = 1;
1644	nextdep = &dep->next;
1645
1646	cur = endp;
1647	}
1648
1649	/* enter the file with its dependencies. */
1650	incdep_record_file (curdep, filename, deps, f);
1651
1652	/* More files? Record them with the same dependency list. */
1653	if (fnnext != fnend)
1654	for (;;)
1655	{
1656	const char *filename_prev = filename;
1657	const char *fnstart;
1658	while (fnnext != fnend && isblank ((unsigned char)*fnnext))
1659	fnnext++;
1660	if (fnnext == fnend)
1661	break;
1662
1663	fnstart = fnnext;
1664	while (fnnext != fnend && !isblank ((unsigned char)*fnnext))
1665	fnnext++;
1666
1667	filename = incdep_dep_strcache (curdep, fnstart, fnnext - fnstart);
1668	if (filename != filename_prev) /* clang optimization. */
1669	incdep_record_file (curdep, filename, incdep_dup_dep_list (curdep, deps), f);
1670	}
1671	}
1672	}
1673	}
1674
1675	/* free the file data */
1676	incdep_xfree (curdep, curdep->file_base);
1677	curdep->file_base = curdep->file_end = NULL;
1678	}
1679
1680	/* Flushes the incdep todo and done lists. */
1681	static void
1682	incdep_flush_it (struct floc *f)
1683	{
1684	incdep_lock ();
1685	for (;;)
1686	{
1687	struct incdep *cur = incdep_head_done;
1688
1689	/* if the done list is empty, grab a todo list entry. */
1690	if (!cur && incdep_head_todo)
1691	{
1692	cur = incdep_head_todo;
1693	if (cur->next)
1694	incdep_head_todo = cur->next;
1695	else
1696	incdep_head_todo = incdep_tail_todo = NULL;
1697	incdep_unlock ();
1698
1699	incdep_read_file (cur, f);
1700	eval_include_dep_file (cur, f);
1701	incdep_freeit (cur);
1702
1703	incdep_lock ();
1704	continue;
1705	}
1706
1707	/* if the todo list and done list are empty we're either done
1708	or will have to wait for the thread(s) to finish. */
1709	if (!cur && !incdep_num_reading)
1710	break; /* done */
1711	if (!cur)
1712	{
1713	while (!incdep_head_done)
1714	incdep_wait_done ();
1715	cur = incdep_head_done;
1716	}
1717
1718	/* we grab the entire done list and work thru it. */
1719	incdep_head_done = incdep_tail_done = NULL;
1720	incdep_unlock ();
1721
1722	while (cur)
1723	{
1724	struct incdep *next = cur->next;
1725	#ifdef PARSE_IN_WORKER
1726	incdep_flush_recorded_instructions (cur);
1727	#else
1728	eval_include_dep_file (cur, f);
1729	#endif
1730	incdep_freeit (cur);
1731	cur = next;
1732	}
1733
1734	incdep_lock ();
1735	} /* outer loop */
1736	incdep_unlock ();
1737	}
1738
1739
1740	/* splits up a list of file names and feeds it to eval_include_dep_file,
1741	employing threads to try speed up the file reading. */
1742	void
1743	eval_include_dep (const char names, struct floc f, enum incdep_op op)
1744	{
1745	struct incdep *head = 0;
1746	struct incdep *tail = 0;
1747	struct incdep *cur;
1748	const char *names_iterator = names;
1749	const char *name;
1750	unsigned int name_len;
1751
1752	/* loop through NAMES, creating a todo list out of them. */
1753
1754	while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1755	{
1756	cur = xmalloc (sizeof (cur) + name_len); / not incdep_xmalloc here */
1757	cur->file_base = cur->file_end = NULL;
1758	memcpy (cur->name, name, name_len);
1759	cur->name[name_len] = '\0';
1760	cur->worker_tid = -1;
1761	#ifdef PARSE_IN_WORKER
1762	cur->err_line_no = 0;
1763	cur->err_msg = NULL;
1764	cur->recorded_variables_in_set_head = NULL;
1765	cur->recorded_variables_in_set_tail = NULL;
1766	cur->recorded_variable_defs_head = NULL;
1767	cur->recorded_variable_defs_tail = NULL;
1768	cur->recorded_file_head = NULL;
1769	cur->recorded_file_tail = NULL;
1770	#endif
1771
1772	cur->next = NULL;
1773	if (tail)
1774	tail->next = cur;
1775	else
1776	head = cur;
1777	tail = cur;
1778	}
1779
1780	#ifdef ELECTRIC_HEAP
1781	if (1)
1782	#else
1783	if (op == incdep_read_it)
1784	#endif
1785	{
1786	/* work our way thru the files directly */
1787
1788	cur = head;
1789	while (cur)
1790	{
1791	struct incdep *next = cur->next;
1792	incdep_read_file (cur, f);
1793	eval_include_dep_file (cur, f);
1794	incdep_freeit (cur);
1795	cur = next;
1796	}
1797	}
1798	else
1799	{
1800	/* initialize the worker threads and related stuff the first time around. */
1801
1802	if (!incdep_initialized)
1803	incdep_init (f);
1804
1805	/* queue the files and notify the worker threads. */
1806
1807	incdep_lock ();
1808
1809	if (incdep_tail_todo)
1810	incdep_tail_todo->next = head;
1811	else
1812	incdep_head_todo = head;
1813	incdep_tail_todo = tail;
1814
1815	incdep_signal_todo ();
1816	incdep_unlock ();
1817
1818	/* flush the todo queue if we're requested to do so. */
1819
1820	if (op == incdep_flush)
1821	incdep_flush_it (f);
1822	}
1823	}
1824
1825	#endif /* CONFIG_WITH_INCLUDEDEP */
1826

Note: See TracBrowser for help on using the repository browser.

source: kBuild/trunk/src/kmk/incdep.c@ 2850

Download in other formats: