VirtualBox

source: kBuild/trunk/src/kmk/incdep.c@ 1864

Last change on this file since 1864 was 1863, checked in by bird, 16 years ago

kmk: Allocation caches for nameseq, dep and idep. next: variable.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 44.3 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1863 2008-10-14 09:46:23Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <[email protected]>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87struct incdep_strcache_entry
88{
89 unsigned int length;
90 unsigned int alignment;
91 unsigned long hash1;
92 unsigned long hash2;
93 char str[1];
94};
95
96struct incdep_variable_in_set
97{
98 struct incdep_variable_in_set *next;
99 /* the parameters */
100 struct incdep_strcache_entry *name_entry;
101 const char *value; /* xmalloc'ed */
102 unsigned int value_length;
103 int duplicate_value; /* 0 */
104 enum variable_origin origin;
105 int recursive;
106 struct variable_set *set;
107 const struct floc *flocp; /* NILF */
108};
109
110struct incdep_variable_def
111{
112 struct incdep_variable_def *next;
113 /* the parameters */
114 const struct floc *flocp; /* NILF */
115 struct incdep_strcache_entry *name_entry;
116 char *value; /* xmalloc'ed, free it */
117 unsigned int value_length;
118 enum variable_origin origin;
119 enum variable_flavor flavor;
120 int target_var;
121};
122
123struct incdep_recorded_files
124{
125 struct incdep_recorded_files *next;
126
127 /* the parameters */
128 struct nameseq *filenames; /* One only, its name is a strcache entry. */
129 const char *pattern; /* NULL */
130 const char *pattern_percent; /* NULL */
131 struct dep *deps; /* All the names are strcache entries. */
132 unsigned int cmds_started; /* 0 */
133 char *commands; /* NULL */
134 unsigned int commands_idx; /* 0 */
135 int two_colon; /* 0 */
136 const struct floc *flocp; /* NILF */
137};
138
139
140/* per dep file structure. */
141struct incdep
142{
143 struct incdep *next;
144 char *file_base;
145 char *file_end;
146
147 int worker_tid;
148#ifdef PARSE_IN_WORKER
149 unsigned int err_line_no;
150 const char *err_msg;
151
152 struct incdep_variable_in_set *recorded_variables_in_set_head;
153 struct incdep_variable_in_set *recorded_variables_in_set_tail;
154
155 struct incdep_variable_def *recorded_variable_defs_head;
156 struct incdep_variable_def *recorded_variable_defs_tail;
157
158 struct incdep_recorded_files *recorded_files_head;
159 struct incdep_recorded_files *recorded_files_tail;
160#endif
161
162 char name[1];
163};
164
165
166/*******************************************************************************
167* Global Variables *
168*******************************************************************************/
169
170/* mutex protecting the globals and an associated condition/event. */
171#ifdef HAVE_PTHREAD
172static pthread_mutex_t incdep_mtx;
173static pthread_cond_t incdep_cond_todo;
174static pthread_cond_t incdep_cond_done;
175
176#elif defined (WINDOWS32)
177static CRITICAL_SECTION incdep_mtx;
178static HANDLE incdep_hev_todo;
179static HANDLE incdep_hev_done;
180static int volatile incdep_hev_todo_waiters;
181static int volatile incdep_hev_done_waiters;
182
183#elif defined (__OS2__)
184static fmutex incdep_mtx;
185static HEV incdep_hev_todo;
186static HEV incdep_hev_done;
187static int volatile incdep_hev_todo_waiters;
188static int volatile incdep_hev_done_waiters;
189#endif
190
191/* flag indicating whether the threads, lock and event/condvars has
192 been initialized or not. */
193static int incdep_initialized;
194
195/* the list of files that needs reading. */
196static struct incdep * volatile incdep_head_todo;
197static struct incdep * volatile incdep_tail_todo;
198
199/* the number of files that are currently being read. */
200static int volatile incdep_num_reading;
201
202/* the list of files that have been read. */
203static struct incdep * volatile incdep_head_done;
204static struct incdep * volatile incdep_tail_done;
205
206
207/* The handles to the worker threads. */
208#ifdef HAVE_PTHREAD
209static pthread_t incdep_threads[1];
210static struct alloccache incdep_dep_caches[1];
211static struct alloccache incdep_nameseq_caches[1];
212
213#elif defined (WINDOWS32)
214static HANDLE incdep_threads[2];
215static struct alloccache incdep_dep_caches[2];
216static struct alloccache incdep_nameseq_caches[2];
217
218#elif defined (__OS2__)
219static TID incdep_threads[2];
220static struct alloccache incdep_dep_caches[2];
221static struct alloccache incdep_nameseq_caches[2];
222#endif
223static unsigned incdep_num_threads;
224
225/* flag indicating whether the worker threads should terminate or not. */
226static int volatile incdep_terminate;
227
228#ifdef __APPLE__
229/* malloc zone for the incdep threads. */
230static malloc_zone_t *incdep_zone;
231#endif
232
233
234/*******************************************************************************
235* Internal Functions *
236*******************************************************************************/
237static void incdep_flush_it (struct floc *);
238static void eval_include_dep_file (struct incdep *, struct floc *);
239
240
241/* xmalloc wrapper.
242 For working around multithreaded performance problems found on Darwin,
243 Linux (glibc), and possibly other systems. */
244static void *
245incdep_xmalloc (struct incdep *cur, size_t size)
246{
247 void *ptr;
248
249#ifdef __APPLE__
250 if (cur && cur->worker_tid != -1)
251 {
252 ptr = malloc_zone_malloc (incdep_zone, size);
253 if (!ptr)
254 fatal (NILF, _("virtual memory exhausted"));
255 }
256 else
257 ptr = xmalloc (size);
258#else
259 ptr = xmalloc (size);
260#endif
261
262 (void)cur;
263 return ptr;
264}
265
266#if 0
267/* memset(malloc(sz),'\0',sz) wrapper. */
268static void *
269incdep_xcalloc (struct incdep *cur, size_t size)
270{
271 void *ptr;
272
273#ifdef __APPLE__
274 if (cur && cur->worker_tid != -1)
275 ptr = malloc_zone_calloc (incdep_zone, size, 1);
276 else
277 ptr = calloc (size, 1);
278#else
279 ptr = calloc (size, 1);
280#endif
281 if (!ptr)
282 fatal (NILF, _("virtual memory exhausted"));
283
284 (void)cur;
285 return ptr;
286}
287#endif /* unused */
288
289/* free wrapper */
290static void
291incdep_xfree (struct incdep *cur, void *ptr)
292{
293 /* free() *must* work for the allocation hacks above because
294 of free_dep_chain. */
295 free (ptr);
296 (void)cur;
297}
298
299/* alloc a nameseq structure. */
300struct nameseq *
301incdep_alloc_nameseq (struct incdep *cur)
302{
303 struct alloccache *cache;
304 if (cur->worker_tid != -1)
305 cache = &incdep_nameseq_caches[cur->worker_tid];
306 else
307 cache = &nameseq_cache;
308 return alloccache_calloc (cache);
309}
310
311/* alloc a dep structure. These are allocated in bunches to save time. */
312struct dep *
313incdep_alloc_dep (struct incdep *cur)
314{
315 struct alloccache *cache;
316 if (cur->worker_tid != -1)
317 cache = &incdep_dep_caches[cur->worker_tid];
318 else
319 cache = &dep_cache;
320 return alloccache_calloc (cache);
321}
322
323/* grow a cache. */
324static void *
325incdep_cache_allocator (void *thrd, unsigned int size)
326{
327 (void)thrd;
328#ifdef __APPLE__
329 return malloc_zone_malloc (incdep_zone, size);
330#else
331 return xmalloc (size);
332#endif
333}
334
335/* acquires the lock */
336void
337incdep_lock(void)
338{
339#ifdef HAVE_PTHREAD
340 pthread_mutex_lock (&incdep_mtx);
341#elif defined (WINDOWS32)
342 EnterCriticalSection (&incdep_mtx);
343#elif defined (__OS2__)
344 _fmutex_request (&incdep_mtx, 0)
345#endif
346}
347
348/* releases the lock */
349void
350incdep_unlock(void)
351{
352#ifdef HAVE_PTHREAD
353 pthread_mutex_unlock (&incdep_mtx);
354#elif defined(WINDOWS32)
355 LeaveCriticalSection (&incdep_mtx);
356#elif defined(__OS2__)
357 _fmutex_release (&incdep_mtx)
358#endif
359}
360
361/* signals the main thread that there is stuff todo. caller owns the lock. */
362static void
363incdep_signal_done (void)
364{
365#ifdef HAVE_PTHREAD
366 pthread_cond_broadcast (&incdep_cond_done);
367#elif defined (WINDOWS32)
368 if (incdep_hev_done_waiters)
369 SetEvent (incdep_hev_done);
370#elif defined (__OS2__)
371 if (incdep_hev_done_waiters)
372 DosPostEventSem (incdep_hev_done);
373#endif
374}
375
376/* waits for a reader to finish reading. caller owns the lock. */
377static void
378incdep_wait_done (void)
379{
380#ifdef HAVE_PTHREAD
381 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
382
383#elif defined (WINDOWS32)
384 ResetEvent (incdep_hev_done);
385 incdep_hev_done_waiters++;
386 incdep_unlock ();
387 WaitForSingleObject (incdep_hev_done, INFINITE);
388 incdep_lock ();
389 incdep_hev_done_waiters--;
390
391#elif defined (__OS2__)
392 ULONG ulIgnore;
393 DosResetEventSem (incdep_hev_done, &ulIgnore);
394 incdep_hev_done_waiters++;
395 incdep_unlock ();
396 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
397 incdep_lock ();
398 incdep_hev_done_waiters--;
399#endif
400}
401
402/* signals the worker threads. caller owns the lock. */
403static void
404incdep_signal_todo (void)
405{
406#ifdef HAVE_PTHREAD
407 pthread_cond_broadcast (&incdep_cond_todo);
408#elif defined(WINDOWS32)
409 if (incdep_hev_todo_waiters)
410 SetEvent (incdep_hev_todo);
411#elif defined(__OS2__)
412 if (incdep_hev_todo_waiters)
413 DosPostEventSem (incdep_hev_todo);
414#endif
415}
416
417/* waits for stuff to arrive in the todo list. caller owns the lock. */
418static void
419incdep_wait_todo (void)
420{
421#ifdef HAVE_PTHREAD
422 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
423
424#elif defined (WINDOWS32)
425 ResetEvent (incdep_hev_todo);
426 incdep_hev_todo_waiters++;
427 incdep_unlock ();
428 WaitForSingleObject (incdep_hev_todo, INFINITE);
429 incdep_lock ();
430 incdep_hev_todo_waiters--;
431
432#elif defined (__OS2__)
433 ULONG ulIgnore;
434 DosResetEventSem (incdep_hev_todo, &ulIgnore);
435 incdep_hev_todo_waiters++;
436 incdep_unlock ();
437 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
438 incdep_lock ();
439 incdep_hev_todo_waiters--;
440#endif
441}
442
443/* Reads a dep file into memory. */
444static int
445incdep_read_file (struct incdep *cur, struct floc *f)
446{
447 int fd;
448 struct stat st;
449
450 errno = 0;
451#ifdef O_BINARY
452 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
453#else
454 fd = open (cur->name, O_RDONLY, 0);
455#endif
456 if (fd < 0)
457 {
458 /* ignore non-existing dependency files. */
459 int err = errno;
460 if (err == ENOENT || stat (cur->name, &st) != 0)
461 return 1;
462 error (f, "%s: %s", cur->name, strerror (err));
463 return -1;
464 }
465 if (!fstat (fd, &st))
466 {
467 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
468 if (read (fd, cur->file_base, st.st_size) == st.st_size)
469 {
470 close (fd);
471 cur->file_end = cur->file_base + st.st_size;
472 cur->file_base[st.st_size] = '\0';
473 return 0;
474 }
475
476 /* bail out */
477
478 error (f, "%s: read: %s", cur->name, strerror (errno));
479 incdep_xfree (cur, cur->file_base);
480 }
481 else
482 error (f, "%s: fstat: %s", cur->name, strerror (errno));
483
484 close (fd);
485 cur->file_base = cur->file_end = NULL;
486 return -1;
487}
488
489/* Free the incdep structure. */
490static void
491incdep_freeit (struct incdep *cur)
492{
493#ifdef PARSE_IN_WORKER
494 assert (!cur->recorded_variables_in_set_head);
495 assert (!cur->recorded_variable_defs_head);
496 assert (!cur->recorded_files_head);
497#endif
498
499 incdep_xfree (cur, cur->file_base);
500 free (cur);
501}
502
503/* A worker thread. */
504void
505incdep_worker (int thrd)
506{
507 incdep_lock ();
508
509 while (!incdep_terminate)
510 {
511 /* get job from the todo list. */
512
513 struct incdep *cur = incdep_head_todo;
514 if (!cur)
515 {
516 incdep_wait_todo ();
517 continue;
518 }
519 if (cur->next)
520 incdep_head_todo = cur->next;
521 else
522 incdep_head_todo = incdep_tail_todo = NULL;
523 incdep_num_reading++;
524
525 /* read the file. */
526
527 incdep_unlock ();
528 cur->worker_tid = thrd;
529
530 incdep_read_file (cur, NILF);
531#ifdef PARSE_IN_WORKER
532 eval_include_dep_file (cur, NILF);
533#endif
534
535 cur->worker_tid = -1;
536 incdep_lock ();
537
538 /* insert finished job into the done list. */
539
540 incdep_num_reading--;
541 cur->next = NULL;
542 if (incdep_tail_done)
543 incdep_tail_done->next = cur;
544 else
545 incdep_head_done = cur;
546 incdep_tail_done = cur;
547
548 incdep_signal_done ();
549 }
550
551 incdep_unlock ();
552}
553
554/* Thread library specific thread functions wrapping incdep_wroker. */
555#ifdef HAVE_PTHREAD
556static void *
557incdep_worker_pthread (void *thrd)
558{
559 incdep_worker ((size_t)thrd);
560 return NULL;
561}
562
563#elif defined (WINDOWS32)
564static unsigned __stdcall
565incdep_worker_windows (void *thrd)
566{
567 incdep_worker ((size_t)thrd);
568 return 0;
569}
570
571#elif defined (__OS2__)
572static void
573incdep_worker_os2 (void *thrd)
574{
575 incdep_worker ((size_t)thrd);
576}
577#endif
578
579/* Creates the the worker threads. */
580static void
581incdep_init (struct floc *f)
582{
583 unsigned i;
584#ifdef HAVE_PTHREAD
585 int rc;
586 pthread_attr_t attr;
587
588#elif defined (WINDOWS32)
589 unsigned tid;
590 uintptr_t hThread;
591
592#elif defined (__OS2__)
593 int rc;
594 int tid;
595#endif
596
597 /* heap hacks */
598
599#ifdef __APPLE__
600 incdep_zone = malloc_create_zone (0, 0);
601 if (!incdep_zone)
602 incdep_zone = malloc_default_zone ();
603#endif
604
605
606 /* create the mutex and two condition variables / event objects. */
607
608#ifdef HAVE_PTHREAD
609 rc = pthread_mutex_init (&incdep_mtx, NULL);
610 if (rc)
611 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
612 rc = pthread_cond_init (&incdep_cond_todo, NULL);
613 if (rc)
614 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
615 rc = pthread_cond_init (&incdep_cond_done, NULL);
616 if (rc)
617 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
618
619#elif defined (WINDOWS32)
620 InitializeCriticalSection (&incdep_mtx);
621 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
622 if (!incdep_hev_todo)
623 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
624 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
625 if (!incdep_hev_done)
626 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
627 incdep_hev_todo_waiters = 0;
628 incdep_hev_done_waiters = 0;
629
630#elif defined (__OS2__)
631 _fmutex_create (&incdep_mtx, 0)
632 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
633 if (rc)
634 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
635 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
636 if (rc)
637 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
638 incdep_hev_todo_waiters = 0;
639 incdep_hev_done_waiters = 0;
640#endif
641
642 /* create the worker threads. */
643
644 incdep_terminate = 0;
645 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
646 if (incdep_num_threads + 1 > job_slots)
647 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
648 for (i = 0; i < incdep_num_threads; i++)
649 {
650 alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
651 incdep_cache_allocator, (void *)i);
652 alloccache_init (&incdep_nameseq_caches[i], sizeof(struct nameseq),
653 "incdep nameseq", incdep_cache_allocator, (void *)i);
654
655#ifdef HAVE_PTHREAD
656 rc = pthread_attr_init (&attr);
657 if (rc)
658 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
659 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
660 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
661 if (rc)
662 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
663 rc = pthread_create(&incdep_threads[i], &attr,
664 incdep_worker_pthread, (void *)i);
665 if (rc)
666 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
667 pthread_attr_destroy (&attr);
668
669#elif defined (WINDOWS32)
670 tid = 0;
671 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
672 (void *)i, 0, &tid);
673 if (hThread == 0 || hThread == ~(uintptr_t)0)
674 fatal (f, _("_beginthreadex failed: err=%d"), errno);
675 incdep_threads[i] = (HANDLE)hThread;
676
677#elif defined (__OS2__)
678 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
679 if (tid <= 0)
680 fatal (f, _("_beginthread failed: err=%d"), errno);
681 incdep_threads[i] = tid;
682#endif
683 }
684
685 incdep_initialized = 1;
686}
687
688/* Flushes outstanding work and terminates the worker threads.
689 This is called from snap_deps(). */
690void
691incdep_flush_and_term (void)
692{
693 unsigned i;
694
695 if (!incdep_initialized)
696 return;
697
698 /* flush any out standing work */
699
700 incdep_flush_it (NILF);
701
702 /* tell the threads to terminate */
703
704 incdep_lock ();
705 incdep_terminate = 1;
706 incdep_signal_todo ();
707 incdep_unlock ();
708
709 /* wait for the threads to quit */
710
711 for (i = 0; i < incdep_num_threads; i++)
712 {
713 /* more later? */
714
715 alloccache_join (&dep_cache, &incdep_dep_caches[i]);
716 alloccache_join (&nameseq_cache, &incdep_nameseq_caches[i]);
717 }
718 incdep_num_threads = 0;
719
720 /* destroy the lock and condition variables / event objects. */
721
722 /* later */
723
724 incdep_initialized = 0;
725}
726
727#ifdef PARSE_IN_WORKER
728/* Flushes a strcache entry returning the actual string cache entry.
729 The input is freed! */
730static const char *
731incdep_flush_strcache_entry (const void *pv_entry)
732{
733 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
734 const char *result;
735 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
736 free (entry);
737 return result;
738}
739
740/* Flushes the recorded instructions. */
741static void
742incdep_flush_recorded_instructions (struct incdep *cur)
743{
744 struct incdep_variable_in_set *rec_vis;
745 struct incdep_variable_def *rec_vd;
746 struct incdep_recorded_files *rec_f;
747
748 /* define_variable_in_set */
749
750 rec_vis = cur->recorded_variables_in_set_head;
751 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
752 if (rec_vis)
753 do
754 {
755 void *free_me = rec_vis;
756 unsigned int name_length = rec_vis->name_entry->length;
757 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
758 name_length,
759 rec_vis->value,
760 rec_vis->value_length,
761 rec_vis->duplicate_value,
762 rec_vis->origin,
763 rec_vis->recursive,
764 rec_vis->set,
765 rec_vis->flocp);
766 rec_vis = rec_vis->next;
767 incdep_xfree (cur, free_me);
768 }
769 while (rec_vis);
770
771 /* do_variable_definition */
772
773 rec_vd = cur->recorded_variable_defs_head;
774 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
775 if (rec_vd)
776 do
777 {
778 void *free_me = rec_vd;
779 do_variable_definition_2 (rec_vd->flocp,
780 incdep_flush_strcache_entry (rec_vd->name_entry),
781 rec_vd->value,
782 rec_vd->value_length,
783 0,
784 rec_vd->value,
785 rec_vd->origin,
786 rec_vd->flavor,
787 rec_vd->target_var);
788 rec_vd = rec_vd->next;
789 incdep_xfree (cur, free_me);
790 }
791 while (rec_vd);
792
793 /* record_files */
794
795 rec_f = cur->recorded_files_head;
796 cur->recorded_files_head = cur->recorded_files_tail = NULL;
797 if (rec_f)
798 do
799 {
800 void *free_me = rec_f;
801 struct dep *dep;
802
803 for (dep = rec_f->deps; dep; dep = dep->next)
804 dep->name = incdep_flush_strcache_entry (dep->name);
805 rec_f->filenames->name = incdep_flush_strcache_entry (rec_f->filenames->name);
806
807 record_files (rec_f->filenames,
808 rec_f->pattern,
809 rec_f->pattern_percent,
810 rec_f->deps,
811 rec_f->cmds_started,
812 rec_f->commands,
813 rec_f->commands_idx,
814 rec_f->two_colon,
815 rec_f->flocp);
816
817 rec_f = rec_f->next;
818 incdep_xfree (cur, free_me);
819 }
820 while (rec_f);
821}
822#endif /* PARSE_IN_WORKER */
823
824/* Record / issue a warning about a misformed dep file. */
825static void
826incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
827{
828 if (cur->worker_tid == -1)
829 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
830#ifdef PARSE_IN_WORKER
831 else
832 {
833 cur->err_line_no = line_no;
834 cur->err_msg = msg;
835 }
836#endif
837}
838
839/* Record / execute a strcache add. */
840static const char *
841incdep_record_strcache (struct incdep *cur, const char *str, int len)
842{
843 const char *ret;
844 if (cur->worker_tid == -1)
845 {
846 /* Make sure the string is terminated before we hand it to
847 strcache_add_len so it does have to make a temporary copy
848 of it on the stack. */
849 char ch = str[len];
850 ((char *)str)[len] = '\0';
851 ret = strcache_add_len (str, len);
852 ((char *)str)[len] = ch;
853 }
854 else
855 {
856 /* Allocate a strcache record for it, pre-hashing the string to save
857 time later on in the main thread. */
858 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
859 memcpy (entry->str, str, len);
860 entry->str[len] = '\0';
861 entry->length = len;
862 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
863
864 ret = (const char *)entry;
865 }
866 return ret;
867}
868
869/* Record / perform a variable definition in a set.
870 The NAME is in the string cache.
871 The VALUE is on the heap.
872 The DUPLICATE_VALUE is always 0. */
873static void
874incdep_record_variable_in_set (struct incdep *cur,
875 const char *name, unsigned int name_length,
876 const char *value,
877 unsigned int value_length,
878 int duplicate_value,
879 enum variable_origin origin,
880 int recursive,
881 struct variable_set *set,
882 const struct floc *flocp)
883{
884 assert (!duplicate_value);
885 if (cur->worker_tid == -1)
886 define_variable_in_set (name, name_length, value, value_length,
887 duplicate_value, origin, recursive, set, flocp);
888#ifdef PARSE_IN_WORKER
889 else
890 {
891 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
892 rec->name_entry = (struct incdep_strcache_entry *)name;
893 rec->value = value;
894 rec->value_length = value_length;
895 rec->duplicate_value = duplicate_value;
896 rec->origin = origin;
897 rec->recursive = recursive;
898 rec->set = set;
899 rec->flocp = flocp;
900
901 rec->next = NULL;
902 if (cur->recorded_variables_in_set_tail)
903 cur->recorded_variables_in_set_tail->next = rec;
904 else
905 cur->recorded_variables_in_set_head = rec;
906 cur->recorded_variables_in_set_tail = rec;
907 }
908#endif
909}
910
911/* Record / perform a variable definition. The VALUE should be disposed of. */
912static void
913incdep_record_variable_def (struct incdep *cur,
914 const struct floc *flocp,
915 const char *name,
916 unsigned int name_length,
917 char *value,
918 unsigned int value_length,
919 enum variable_origin origin,
920 enum variable_flavor flavor,
921 int target_var)
922{
923 if (cur->worker_tid == -1)
924 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
925 origin, flavor, target_var);
926#ifdef PARSE_IN_WORKER
927 else
928 {
929 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
930 rec->flocp = flocp;
931 rec->name_entry = (struct incdep_strcache_entry *)name;
932 rec->value = value;
933 rec->value_length = value_length;
934 rec->origin = origin;
935 rec->flavor = flavor;
936 rec->target_var = target_var;
937
938 rec->next = NULL;
939 if (cur->recorded_variable_defs_tail)
940 cur->recorded_variable_defs_tail->next = rec;
941 else
942 cur->recorded_variable_defs_head = rec;
943 cur->recorded_variable_defs_tail = rec;
944 }
945#else
946 (void)name_length;
947#endif
948}
949
950/* Record files.*/
951static void
952incdep_record_files (struct incdep *cur,
953 struct nameseq *filenames, const char *pattern,
954 const char *pattern_percent, struct dep *deps,
955 unsigned int cmds_started, char *commands,
956 unsigned int commands_idx, int two_colon,
957 const struct floc *flocp)
958{
959 if (cur->worker_tid == -1)
960 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
961 commands, commands_idx, two_colon, flocp);
962#ifdef PARSE_IN_WORKER
963 else
964 {
965 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
966
967 rec->filenames = filenames;
968 rec->pattern = pattern;
969 rec->pattern_percent = pattern_percent;
970 rec->deps = deps;
971 rec->cmds_started = cmds_started;
972 rec->commands = commands;
973 rec->commands_idx = commands_idx;
974 rec->two_colon = two_colon;
975 rec->flocp = flocp;
976
977 rec->next = NULL;
978 if (cur->recorded_files_tail)
979 cur->recorded_files_tail->next = rec;
980 else
981 cur->recorded_files_head = rec;
982 cur->recorded_files_tail = rec;
983 }
984#endif
985}
986
987
988/* no nonsense dependency file including.
989
990 Because nobody wants bogus dependency files to break their incremental
991 builds with hard to comprehend error messages, this function does not
992 use the normal eval routine but does all the parsing itself. This isn't,
993 as much work as it sounds, because the necessary feature set is very
994 limited.
995
996 eval_include_dep_file groks:
997
998 define var
999 endef
1000
1001 var [|:|?|>]= value [\]
1002
1003 [\]
1004 file: [deps] [\]
1005
1006 */
1007static void
1008eval_include_dep_file (struct incdep *curdep, struct floc *f)
1009{
1010 unsigned line_no = 1;
1011 const char *file_end = curdep->file_end;
1012 const char *cur = curdep->file_base;
1013 const char *endp;
1014
1015 /* if no file data, just return immediately. */
1016 if (!cur)
1017 return;
1018
1019 /* now parse the file. */
1020 while (cur < file_end)
1021 {
1022 /* skip empty lines */
1023 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1024 ++cur;
1025 if (cur >= file_end)
1026 break;
1027 if (*cur == '#')
1028 {
1029 cur = memchr (cur, '\n', file_end - cur);
1030 if (!cur)
1031 break;
1032 }
1033 if (*cur == '\\')
1034 {
1035 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1036 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1037 : (file_end - cur == 1) ? 1 : 0;
1038 if (eol_len)
1039 {
1040 cur += eol_len;
1041 line_no++;
1042 continue;
1043 }
1044 }
1045 if (*cur == '\n')
1046 {
1047 cur++;
1048 line_no++;
1049 continue;
1050 }
1051
1052 /* define var
1053 ...
1054 endef */
1055 if (strneq (cur, "define ", 7))
1056 {
1057 const char *var;
1058 unsigned var_len;
1059 const char *value_start;
1060 const char *value_end;
1061 char *value;
1062 unsigned value_len;
1063 int found_endef = 0;
1064
1065 /* extract the variable name. */
1066 cur += 7;
1067 while (isblank ((unsigned char)*cur))
1068 ++cur;
1069 value_start = endp = memchr (cur, '\n', file_end - cur);
1070 if (!endp)
1071 endp = cur;
1072 while (endp > cur && isspace ((unsigned char)endp[-1]))
1073 --endp;
1074 var_len = endp - cur;
1075 if (!var_len)
1076 {
1077 incdep_warn (curdep, line_no, "bogus define statement.");
1078 break;
1079 }
1080 var = incdep_record_strcache (curdep, cur, var_len);
1081
1082 /* find the end of the variable. */
1083 cur = value_end = value_start = value_start + 1;
1084 ++line_no;
1085 while (cur < file_end)
1086 {
1087 /* check for endef, don't bother with skipping leading spaces. */
1088 if ( file_end - cur >= 5
1089 && strneq (cur, "endef", 5))
1090 {
1091 endp = cur + 5;
1092 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1093 endp++;
1094 if (endp >= file_end || *endp == '\n')
1095 {
1096 found_endef = 1;
1097 cur = endp >= file_end ? file_end : endp + 1;
1098 break;
1099 }
1100 }
1101
1102 /* skip a line ahead. */
1103 cur = value_end = memchr (cur, '\n', file_end - cur);
1104 if (cur != NULL)
1105 ++cur;
1106 else
1107 cur = value_end = file_end;
1108 ++line_no;
1109 }
1110
1111 if (!found_endef)
1112 {
1113 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1114 break;
1115 }
1116 value_len = value_end - value_start;
1117 if (memchr (value_start, '\0', value_len))
1118 {
1119 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1120 break;
1121 }
1122
1123 /* make a copy of the value, converting \r\n to \n, and define it. */
1124 value = incdep_xmalloc (curdep, value_len + 1);
1125 endp = memchr (value_start, '\r', value_len);
1126 if (endp)
1127 {
1128 const char *src = value_start;
1129 char *dst = value;
1130 for (;;)
1131 {
1132 size_t len = endp - src;
1133 memcpy (dst, src, len);
1134 dst += len;
1135 src = endp;
1136 if (src + 1 < file_end && src[1] == '\n')
1137 src++; /* skip the '\r' */
1138 if (src >= value_end)
1139 break;
1140 endp = memchr (endp + 1, '\r', src - value_end);
1141 if (!endp)
1142 endp = value_end;
1143 }
1144 value_len = dst - value;
1145 }
1146 else
1147 memcpy (value, value_start, value_len);
1148 value [value_len] = '\0';
1149
1150 incdep_record_variable_in_set (curdep,
1151 var, var_len, value, value_len,
1152 0 /* don't duplicate */, o_file,
1153 0 /* defines are recursive but this is faster */,
1154 NULL /* global set */, f);
1155 }
1156
1157 /* file: deps
1158 OR
1159 variable [:]= value */
1160 else
1161 {
1162 const char *colonp;
1163 const char *equalp;
1164
1165 /* Look for a colon and an equal sign, optimize for colon.
1166 Only one file is support and the colon / equal must be on
1167 the same line. */
1168 colonp = memchr (cur, ':', file_end - cur);
1169#ifdef HAVE_DOS_PATHS
1170 while ( colonp
1171 && colonp + 1 < file_end
1172 && (colonp[1] == '/' || colonp[1] == '\\')
1173 && colonp > cur
1174 && isalpha ((unsigned char)colonp[-1])
1175 && ( colonp == cur + 1
1176 || strchr (" \t(", colonp[-2]) != 0))
1177 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1178#endif
1179 endp = NULL;
1180 if ( !colonp
1181 || (endp = memchr (cur, '\n', colonp - cur)))
1182 {
1183 colonp = NULL;
1184 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1185 if ( !equalp
1186 || (!endp && memchr (cur, '\n', equalp - cur)))
1187 {
1188 incdep_warn (curdep, line_no, "no colon.");
1189 break;
1190 }
1191 }
1192 else
1193 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1194 ? colonp + 2 : file_end) - cur);
1195 if (equalp)
1196 {
1197 /* An assignment of some sort. */
1198 const char *var;
1199 unsigned var_len;
1200 const char *value_start;
1201 const char *value_end;
1202 char *value;
1203 unsigned value_len;
1204 unsigned multi_line = 0;
1205 enum variable_flavor flavor;
1206
1207 /* figure the flavor first. */
1208 flavor = f_recursive;
1209 if (equalp > cur)
1210 {
1211 if (equalp[-1] == ':')
1212 flavor = f_simple;
1213 else if (equalp[-1] == '?')
1214 flavor = f_conditional;
1215 else if (equalp[-1] == '+')
1216 flavor = f_append;
1217 else if (equalp[-1] == '>')
1218 flavor = f_prepend;
1219 }
1220
1221 /* extract the variable name. */
1222 endp = flavor == f_recursive ? equalp : equalp - 1;
1223 while (endp > cur && isblank ((unsigned char)endp[-1]))
1224 --endp;
1225 var_len = endp - cur;
1226 if (!var_len)
1227 {
1228 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1229 break;
1230 }
1231 if ( memchr (cur, '$', var_len)
1232 || memchr (cur, ' ', var_len)
1233 || memchr (cur, '\t', var_len))
1234 {
1235 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1236 break;
1237 }
1238 var = incdep_record_strcache (curdep, cur, var_len);
1239
1240 /* find the start of the value. */
1241 cur = equalp + 1;
1242 while (cur < file_end && isblank ((unsigned char)*cur))
1243 cur++;
1244 value_start = cur;
1245
1246 /* find the end of the value / line (this isn't 101% correct). */
1247 value_end = cur;
1248 while (cur < file_end)
1249 {
1250 endp = value_end = memchr (cur, '\n', file_end - cur);
1251 if (!value_end)
1252 value_end = file_end;
1253 if (value_end - 1 >= cur && value_end[-1] == '\r')
1254 --value_end;
1255 if (value_end - 1 < cur || value_end[-1] != '\\')
1256 {
1257 cur = endp ? endp + 1 : file_end;
1258 break;
1259 }
1260 --value_end;
1261 if (value_end - 1 >= cur && value_end[-1] == '\\')
1262 {
1263 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1264 cur = NULL;
1265 break;
1266 }
1267 if (!endp)
1268 {
1269 cur = file_end;
1270 break;
1271 }
1272
1273 cur = endp + 1;
1274 ++multi_line;
1275 ++line_no;
1276 }
1277 if (!cur)
1278 break;
1279 ++line_no;
1280
1281 /* make a copy of the value, converting \r\n to \n, and define it. */
1282 value_len = value_end - value_start;
1283 value = incdep_xmalloc (curdep, value_len + 1);
1284 if (!multi_line)
1285 memcpy (value, value_start, value_len);
1286 else
1287 {
1288 /* unescape it */
1289 const char *src = value_start;
1290 char *dst = value;
1291 while (src < value_end)
1292 {
1293 const char *nextp;
1294
1295 endp = memchr (src, '\n', value_end - src);
1296 if (!endp)
1297 nextp = endp = value_end;
1298 else
1299 nextp = endp + 1;
1300 if (endp > src && endp[-1] == '\r')
1301 --endp;
1302 if (endp > src && endp[-1] == '\\')
1303 --endp;
1304
1305 if (src != value_start)
1306 *dst++ = ' ';
1307 memcpy (dst, src, endp - src);
1308 dst += endp - src;
1309 src = nextp;
1310 }
1311 value_len = dst - value;
1312 }
1313 value [value_len] = '\0';
1314
1315 /* do the definition */
1316 if (flavor == f_recursive
1317 || ( flavor == f_simple
1318 && !memchr (value, '$', value_len)))
1319 incdep_record_variable_in_set (curdep,
1320 var, var_len, value, value_len,
1321 0 /* don't duplicate */, o_file,
1322 flavor == f_recursive /* recursive */,
1323 NULL /* global set */, f);
1324 else
1325 incdep_record_variable_def (curdep,
1326 f, var, var_len, value, value_len,
1327 o_file, flavor, 0 /* not target var */);
1328 }
1329 else
1330 {
1331 /* file: dependencies */
1332
1333 struct nameseq *filenames = 0;
1334 struct dep *deps = 0;
1335 struct dep **nextdep = &deps;
1336 struct dep *dep;
1337
1338 /* extract the filename, ASSUME a single one. */
1339 endp = colonp;
1340 while (endp > cur && isblank ((unsigned char)endp[-1]))
1341 --endp;
1342 if (cur == endp)
1343 {
1344 incdep_warn (curdep, line_no, "empty filename.");
1345 break;
1346 }
1347 if ( memchr (cur, '$', endp - cur)
1348 || memchr (cur, ' ', endp - cur)
1349 || memchr (cur, '\t', endp - cur))
1350 {
1351 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1352 break;
1353 }
1354 filenames = incdep_alloc_nameseq (curdep);
1355 memset (filenames, 0, sizeof (*filenames));
1356 filenames->name = incdep_record_strcache (curdep, cur, endp - cur);
1357
1358 /* parse any dependencies. */
1359 cur = colonp + 1;
1360 while (cur < file_end)
1361 {
1362 /* skip blanks and count lines. */
1363 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1364 ++cur;
1365 if (cur >= file_end)
1366 break;
1367 if (*cur == '\n')
1368 {
1369 cur++;
1370 line_no++;
1371 break;
1372 }
1373
1374 /* continuation + eol? */
1375 if (*cur == '\\')
1376 {
1377 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1378 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1379 : (file_end - cur == 1) ? 1 : 0;
1380 if (eol_len)
1381 {
1382 cur += eol_len;
1383 line_no++;
1384 continue;
1385 }
1386 }
1387
1388 /* find the end of the filename */
1389 endp = cur;
1390 while (endp < file_end && !isspace ((unsigned char)*endp))
1391 ++endp;
1392
1393 /* add it to the list. */
1394 *nextdep = dep = incdep_alloc_dep (curdep);
1395 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1396 dep->includedep = 1;
1397 nextdep = &dep->next;
1398
1399 cur = endp;
1400 }
1401
1402 /* enter the file with its dependencies. */
1403 incdep_record_files (curdep,
1404 filenames, NULL, NULL, deps, 0, NULL, 0, 0, f);
1405 }
1406 }
1407 }
1408
1409 /* free the file data */
1410 incdep_xfree (curdep, curdep->file_base);
1411 curdep->file_base = curdep->file_end = NULL;
1412}
1413
1414/* Flushes the incdep todo and done lists. */
1415static void
1416incdep_flush_it (struct floc *f)
1417{
1418 incdep_lock ();
1419 for (;;)
1420 {
1421 struct incdep *cur = incdep_head_done;
1422
1423 /* if the done list is empty, grab a todo list entry. */
1424 if (!cur && incdep_head_todo)
1425 {
1426 cur = incdep_head_todo;
1427 if (cur->next)
1428 incdep_head_todo = cur->next;
1429 else
1430 incdep_head_todo = incdep_tail_todo = NULL;
1431 incdep_unlock ();
1432
1433 incdep_read_file (cur, f);
1434 eval_include_dep_file (cur, f);
1435 incdep_freeit (cur);
1436
1437 incdep_lock ();
1438 continue;
1439 }
1440
1441 /* if the todo list and done list are empty we're either done
1442 or will have to wait for the thread(s) to finish. */
1443 if (!cur && !incdep_num_reading)
1444 break; /* done */
1445 if (!cur)
1446 {
1447 while (!incdep_head_done)
1448 incdep_wait_done ();
1449 cur = incdep_head_done;
1450 }
1451
1452 /* we grab the entire done list and work thru it. */
1453 incdep_head_done = incdep_tail_done = NULL;
1454 incdep_unlock ();
1455
1456 while (cur)
1457 {
1458 struct incdep *next = cur->next;
1459#ifdef PARSE_IN_WORKER
1460 incdep_flush_recorded_instructions (cur);
1461#else
1462 eval_include_dep_file (cur, f);
1463#endif
1464 incdep_freeit (cur);
1465 cur = next;
1466 }
1467
1468 incdep_lock ();
1469 } /* outer loop */
1470 incdep_unlock ();
1471}
1472
1473
1474/* splits up a list of file names and feeds it to eval_include_dep_file,
1475 employing threads to try speed up the file reading. */
1476void
1477eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1478{
1479 struct incdep *head = 0;
1480 struct incdep *tail = 0;
1481 struct incdep *cur;
1482 const char *names_iterator = names;
1483 const char *name;
1484 unsigned int name_len;
1485
1486 /* loop through NAMES, creating a todo list out of them. */
1487
1488 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1489 {
1490 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1491 cur->file_base = cur->file_end = NULL;
1492 memcpy (cur->name, name, name_len);
1493 cur->name[name_len] = '\0';
1494 cur->worker_tid = -1;
1495#ifdef PARSE_IN_WORKER
1496 cur->err_line_no = 0;
1497 cur->err_msg = NULL;
1498 cur->recorded_variables_in_set_head = NULL;
1499 cur->recorded_variables_in_set_tail = NULL;
1500 cur->recorded_variable_defs_head = NULL;
1501 cur->recorded_variable_defs_tail = NULL;
1502 cur->recorded_files_head = NULL;
1503 cur->recorded_files_tail = NULL;
1504#endif
1505
1506 cur->next = NULL;
1507 if (tail)
1508 tail->next = cur;
1509 else
1510 head = cur;
1511 tail = cur;
1512 }
1513
1514#ifdef ELECTRIC_HEAP
1515 if (1)
1516#else
1517 if (op == incdep_read_it)
1518#endif
1519 {
1520 /* work our way thru the files directly */
1521
1522 cur = head;
1523 while (cur)
1524 {
1525 struct incdep *next = cur->next;
1526 incdep_read_file (cur, f);
1527 eval_include_dep_file (cur, f);
1528 incdep_freeit (cur);
1529 cur = next;
1530 }
1531 }
1532 else
1533 {
1534 /* initialize the worker threads and related stuff the first time around. */
1535
1536 if (!incdep_initialized)
1537 incdep_init (f);
1538
1539 /* queue the files and notify the worker threads. */
1540
1541 incdep_lock ();
1542
1543 if (incdep_tail_todo)
1544 incdep_tail_todo->next = head;
1545 else
1546 incdep_head_todo = head;
1547 incdep_tail_todo = tail;
1548
1549 incdep_signal_todo ();
1550 incdep_unlock ();
1551
1552 /* flush the todo queue if we're requested to do so. */
1553
1554 if (op == incdep_flush)
1555 incdep_flush_it (f);
1556 }
1557}
1558
1559#endif /* CONFIG_WITH_INCLUDEDEP */
1560
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette