VirtualBox

source: kBuild/trunk/src/kmk/incdep.c@ 1865

Last change on this file since 1865 was 1865, checked in by bird, 16 years ago

kmk/incdep: don't allocate nameseq records, there will be a whole lot of them.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Id
File size: 44.5 KB
Line 
1#ifdef CONFIG_WITH_INCLUDEDEP
2/* $Id: incdep.c 1865 2008-10-15 01:08:24Z bird $ */
3/** @file
4 * incdep - Simple dependency files.
5 */
6
7/*
8 * Copyright (c) 2006-2008 knut st. osmundsen <[email protected]>
9 *
10 * This file is part of kBuild.
11 *
12 * kBuild is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * kBuild is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with kBuild; if not, write to the Free Software
24 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 *
26 */
27
28
29/*******************************************************************************
30* Header Files *
31*******************************************************************************/
32#ifdef __OS2__
33# define INCL_BASE
34# define INCL_ERRORS
35#endif
36
37#include "make.h"
38
39#if !defined(WINDOWS32) && !defined(__OS2__)
40# define HAVE_PTHREAD
41#endif
42
43#include <assert.h>
44
45#include <glob.h>
46
47#include "dep.h"
48#include "filedef.h"
49#include "job.h"
50#include "commands.h"
51#include "variable.h"
52#include "rule.h"
53#include "debug.h"
54#include "hash.h"
55
56#ifdef HAVE_FCNTL_H
57# include <fcntl.h>
58#else
59# include <sys/file.h>
60#endif
61
62#ifdef WINDOWS32
63# include <io.h>
64# include <process.h>
65# include <Windows.h>
66# define PARSE_IN_WORKER
67#endif
68
69#ifdef __OS2__
70# include <os2.h>
71# include <sys/fmutex.h>
72#endif
73
74#ifdef HAVE_PTHREAD
75# include <pthread.h>
76#endif
77
78#ifdef __APPLE__
79# include <malloc/malloc.h>
80# define PARSE_IN_WORKER
81#endif
82
83
84/*******************************************************************************
85* Structures and Typedefs *
86*******************************************************************************/
87struct incdep_strcache_entry
88{
89 unsigned int length;
90 unsigned int alignment;
91 unsigned long hash1;
92 unsigned long hash2;
93 char str[1];
94};
95
96struct incdep_variable_in_set
97{
98 struct incdep_variable_in_set *next;
99 /* the parameters */
100 struct incdep_strcache_entry *name_entry;
101 const char *value; /* xmalloc'ed */
102 unsigned int value_length;
103 int duplicate_value; /* 0 */
104 enum variable_origin origin;
105 int recursive;
106 struct variable_set *set;
107 const struct floc *flocp; /* NILF */
108};
109
110struct incdep_variable_def
111{
112 struct incdep_variable_def *next;
113 /* the parameters */
114 const struct floc *flocp; /* NILF */
115 struct incdep_strcache_entry *name_entry;
116 char *value; /* xmalloc'ed, free it */
117 unsigned int value_length;
118 enum variable_origin origin;
119 enum variable_flavor flavor;
120 int target_var;
121};
122
123struct incdep_recorded_files
124{
125 struct incdep_recorded_files *next;
126
127 /* the parameters */
128 struct incdep_strcache_entry *filename_entry; /* Converted to a nameseq record. */
129 const char *pattern; /* NULL */
130 const char *pattern_percent; /* NULL */
131 struct dep *deps; /* All the names are strcache entries. */
132 unsigned int cmds_started; /* 0 */
133 char *commands; /* NULL */
134 unsigned int commands_idx; /* 0 */
135 int two_colon; /* 0 */
136 const struct floc *flocp; /* NILF */
137};
138
139
140/* per dep file structure. */
141struct incdep
142{
143 struct incdep *next;
144 char *file_base;
145 char *file_end;
146
147 int worker_tid;
148#ifdef PARSE_IN_WORKER
149 unsigned int err_line_no;
150 const char *err_msg;
151
152 struct incdep_variable_in_set *recorded_variables_in_set_head;
153 struct incdep_variable_in_set *recorded_variables_in_set_tail;
154
155 struct incdep_variable_def *recorded_variable_defs_head;
156 struct incdep_variable_def *recorded_variable_defs_tail;
157
158 struct incdep_recorded_files *recorded_files_head;
159 struct incdep_recorded_files *recorded_files_tail;
160#endif
161
162 char name[1];
163};
164
165
166/*******************************************************************************
167* Global Variables *
168*******************************************************************************/
169
170/* mutex protecting the globals and an associated condition/event. */
171#ifdef HAVE_PTHREAD
172static pthread_mutex_t incdep_mtx;
173static pthread_cond_t incdep_cond_todo;
174static pthread_cond_t incdep_cond_done;
175
176#elif defined (WINDOWS32)
177static CRITICAL_SECTION incdep_mtx;
178static HANDLE incdep_hev_todo;
179static HANDLE incdep_hev_done;
180static int volatile incdep_hev_todo_waiters;
181static int volatile incdep_hev_done_waiters;
182
183#elif defined (__OS2__)
184static fmutex incdep_mtx;
185static HEV incdep_hev_todo;
186static HEV incdep_hev_done;
187static int volatile incdep_hev_todo_waiters;
188static int volatile incdep_hev_done_waiters;
189#endif
190
191/* flag indicating whether the threads, lock and event/condvars has
192 been initialized or not. */
193static int incdep_initialized;
194
195/* the list of files that needs reading. */
196static struct incdep * volatile incdep_head_todo;
197static struct incdep * volatile incdep_tail_todo;
198
199/* the number of files that are currently being read. */
200static int volatile incdep_num_reading;
201
202/* the list of files that have been read. */
203static struct incdep * volatile incdep_head_done;
204static struct incdep * volatile incdep_tail_done;
205
206
207/* The handles to the worker threads. */
208#ifdef HAVE_PTHREAD
209static pthread_t incdep_threads[1];
210static struct alloccache incdep_dep_caches[1];
211static struct alloccache incdep_nameseq_caches[1];
212
213#elif defined (WINDOWS32)
214static HANDLE incdep_threads[2];
215static struct alloccache incdep_dep_caches[2];
216static struct alloccache incdep_nameseq_caches[2];
217
218#elif defined (__OS2__)
219static TID incdep_threads[2];
220static struct alloccache incdep_dep_caches[2];
221static struct alloccache incdep_nameseq_caches[2];
222#endif
223static unsigned incdep_num_threads;
224
225/* flag indicating whether the worker threads should terminate or not. */
226static int volatile incdep_terminate;
227
228#ifdef __APPLE__
229/* malloc zone for the incdep threads. */
230static malloc_zone_t *incdep_zone;
231#endif
232
233
234/*******************************************************************************
235* Internal Functions *
236*******************************************************************************/
237static void incdep_flush_it (struct floc *);
238static void eval_include_dep_file (struct incdep *, struct floc *);
239
240
241/* xmalloc wrapper.
242 For working around multithreaded performance problems found on Darwin,
243 Linux (glibc), and possibly other systems. */
244static void *
245incdep_xmalloc (struct incdep *cur, size_t size)
246{
247 void *ptr;
248
249#ifdef __APPLE__
250 if (cur && cur->worker_tid != -1)
251 {
252 ptr = malloc_zone_malloc (incdep_zone, size);
253 if (!ptr)
254 fatal (NILF, _("virtual memory exhausted"));
255 }
256 else
257 ptr = xmalloc (size);
258#else
259 ptr = xmalloc (size);
260#endif
261
262 (void)cur;
263 return ptr;
264}
265
266#if 0
267/* memset(malloc(sz),'\0',sz) wrapper. */
268static void *
269incdep_xcalloc (struct incdep *cur, size_t size)
270{
271 void *ptr;
272
273#ifdef __APPLE__
274 if (cur && cur->worker_tid != -1)
275 ptr = malloc_zone_calloc (incdep_zone, size, 1);
276 else
277 ptr = calloc (size, 1);
278#else
279 ptr = calloc (size, 1);
280#endif
281 if (!ptr)
282 fatal (NILF, _("virtual memory exhausted"));
283
284 (void)cur;
285 return ptr;
286}
287#endif /* unused */
288
289/* free wrapper */
290static void
291incdep_xfree (struct incdep *cur, void *ptr)
292{
293 /* free() *must* work for the allocation hacks above because
294 of free_dep_chain. */
295 free (ptr);
296 (void)cur;
297}
298
299/* alloc a nameseq structure. */
300struct nameseq *
301incdep_alloc_nameseq (struct incdep *cur)
302{
303 struct alloccache *cache;
304 if (cur->worker_tid != -1)
305 cache = &incdep_nameseq_caches[cur->worker_tid];
306 else
307 cache = &nameseq_cache;
308 return alloccache_calloc (cache);
309}
310
311/* alloc a dep structure. These are allocated in bunches to save time. */
312struct dep *
313incdep_alloc_dep (struct incdep *cur)
314{
315 struct alloccache *cache;
316 if (cur->worker_tid != -1)
317 cache = &incdep_dep_caches[cur->worker_tid];
318 else
319 cache = &dep_cache;
320 return alloccache_calloc (cache);
321}
322
323/* grow a cache. */
324static void *
325incdep_cache_allocator (void *thrd, unsigned int size)
326{
327 (void)thrd;
328#ifdef __APPLE__
329 return malloc_zone_malloc (incdep_zone, size);
330#else
331 return xmalloc (size);
332#endif
333}
334
335/* acquires the lock */
336void
337incdep_lock(void)
338{
339#ifdef HAVE_PTHREAD
340 pthread_mutex_lock (&incdep_mtx);
341#elif defined (WINDOWS32)
342 EnterCriticalSection (&incdep_mtx);
343#elif defined (__OS2__)
344 _fmutex_request (&incdep_mtx, 0)
345#endif
346}
347
348/* releases the lock */
349void
350incdep_unlock(void)
351{
352#ifdef HAVE_PTHREAD
353 pthread_mutex_unlock (&incdep_mtx);
354#elif defined(WINDOWS32)
355 LeaveCriticalSection (&incdep_mtx);
356#elif defined(__OS2__)
357 _fmutex_release (&incdep_mtx)
358#endif
359}
360
361/* signals the main thread that there is stuff todo. caller owns the lock. */
362static void
363incdep_signal_done (void)
364{
365#ifdef HAVE_PTHREAD
366 pthread_cond_broadcast (&incdep_cond_done);
367#elif defined (WINDOWS32)
368 if (incdep_hev_done_waiters)
369 SetEvent (incdep_hev_done);
370#elif defined (__OS2__)
371 if (incdep_hev_done_waiters)
372 DosPostEventSem (incdep_hev_done);
373#endif
374}
375
376/* waits for a reader to finish reading. caller owns the lock. */
377static void
378incdep_wait_done (void)
379{
380#ifdef HAVE_PTHREAD
381 pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
382
383#elif defined (WINDOWS32)
384 ResetEvent (incdep_hev_done);
385 incdep_hev_done_waiters++;
386 incdep_unlock ();
387 WaitForSingleObject (incdep_hev_done, INFINITE);
388 incdep_lock ();
389 incdep_hev_done_waiters--;
390
391#elif defined (__OS2__)
392 ULONG ulIgnore;
393 DosResetEventSem (incdep_hev_done, &ulIgnore);
394 incdep_hev_done_waiters++;
395 incdep_unlock ();
396 DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
397 incdep_lock ();
398 incdep_hev_done_waiters--;
399#endif
400}
401
402/* signals the worker threads. caller owns the lock. */
403static void
404incdep_signal_todo (void)
405{
406#ifdef HAVE_PTHREAD
407 pthread_cond_broadcast (&incdep_cond_todo);
408#elif defined(WINDOWS32)
409 if (incdep_hev_todo_waiters)
410 SetEvent (incdep_hev_todo);
411#elif defined(__OS2__)
412 if (incdep_hev_todo_waiters)
413 DosPostEventSem (incdep_hev_todo);
414#endif
415}
416
417/* waits for stuff to arrive in the todo list. caller owns the lock. */
418static void
419incdep_wait_todo (void)
420{
421#ifdef HAVE_PTHREAD
422 pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
423
424#elif defined (WINDOWS32)
425 ResetEvent (incdep_hev_todo);
426 incdep_hev_todo_waiters++;
427 incdep_unlock ();
428 WaitForSingleObject (incdep_hev_todo, INFINITE);
429 incdep_lock ();
430 incdep_hev_todo_waiters--;
431
432#elif defined (__OS2__)
433 ULONG ulIgnore;
434 DosResetEventSem (incdep_hev_todo, &ulIgnore);
435 incdep_hev_todo_waiters++;
436 incdep_unlock ();
437 DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
438 incdep_lock ();
439 incdep_hev_todo_waiters--;
440#endif
441}
442
443/* Reads a dep file into memory. */
444static int
445incdep_read_file (struct incdep *cur, struct floc *f)
446{
447 int fd;
448 struct stat st;
449
450 errno = 0;
451#ifdef O_BINARY
452 fd = open (cur->name, O_RDONLY | O_BINARY, 0);
453#else
454 fd = open (cur->name, O_RDONLY, 0);
455#endif
456 if (fd < 0)
457 {
458 /* ignore non-existing dependency files. */
459 int err = errno;
460 if (err == ENOENT || stat (cur->name, &st) != 0)
461 return 1;
462 error (f, "%s: %s", cur->name, strerror (err));
463 return -1;
464 }
465 if (!fstat (fd, &st))
466 {
467 cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
468 if (read (fd, cur->file_base, st.st_size) == st.st_size)
469 {
470 close (fd);
471 cur->file_end = cur->file_base + st.st_size;
472 cur->file_base[st.st_size] = '\0';
473 return 0;
474 }
475
476 /* bail out */
477
478 error (f, "%s: read: %s", cur->name, strerror (errno));
479 incdep_xfree (cur, cur->file_base);
480 }
481 else
482 error (f, "%s: fstat: %s", cur->name, strerror (errno));
483
484 close (fd);
485 cur->file_base = cur->file_end = NULL;
486 return -1;
487}
488
489/* Free the incdep structure. */
490static void
491incdep_freeit (struct incdep *cur)
492{
493#ifdef PARSE_IN_WORKER
494 assert (!cur->recorded_variables_in_set_head);
495 assert (!cur->recorded_variable_defs_head);
496 assert (!cur->recorded_files_head);
497#endif
498
499 incdep_xfree (cur, cur->file_base);
500 free (cur);
501}
502
503/* A worker thread. */
504void
505incdep_worker (int thrd)
506{
507 incdep_lock ();
508
509 while (!incdep_terminate)
510 {
511 /* get job from the todo list. */
512
513 struct incdep *cur = incdep_head_todo;
514 if (!cur)
515 {
516 incdep_wait_todo ();
517 continue;
518 }
519 if (cur->next)
520 incdep_head_todo = cur->next;
521 else
522 incdep_head_todo = incdep_tail_todo = NULL;
523 incdep_num_reading++;
524
525 /* read the file. */
526
527 incdep_unlock ();
528 cur->worker_tid = thrd;
529
530 incdep_read_file (cur, NILF);
531#ifdef PARSE_IN_WORKER
532 eval_include_dep_file (cur, NILF);
533#endif
534
535 cur->worker_tid = -1;
536 incdep_lock ();
537
538 /* insert finished job into the done list. */
539
540 incdep_num_reading--;
541 cur->next = NULL;
542 if (incdep_tail_done)
543 incdep_tail_done->next = cur;
544 else
545 incdep_head_done = cur;
546 incdep_tail_done = cur;
547
548 incdep_signal_done ();
549 }
550
551 incdep_unlock ();
552}
553
554/* Thread library specific thread functions wrapping incdep_wroker. */
555#ifdef HAVE_PTHREAD
556static void *
557incdep_worker_pthread (void *thrd)
558{
559 incdep_worker ((size_t)thrd);
560 return NULL;
561}
562
563#elif defined (WINDOWS32)
564static unsigned __stdcall
565incdep_worker_windows (void *thrd)
566{
567 incdep_worker ((size_t)thrd);
568 return 0;
569}
570
571#elif defined (__OS2__)
572static void
573incdep_worker_os2 (void *thrd)
574{
575 incdep_worker ((size_t)thrd);
576}
577#endif
578
579/* Creates the the worker threads. */
580static void
581incdep_init (struct floc *f)
582{
583 unsigned i;
584#ifdef HAVE_PTHREAD
585 int rc;
586 pthread_attr_t attr;
587
588#elif defined (WINDOWS32)
589 unsigned tid;
590 uintptr_t hThread;
591
592#elif defined (__OS2__)
593 int rc;
594 int tid;
595#endif
596
597 /* heap hacks */
598
599#ifdef __APPLE__
600 incdep_zone = malloc_create_zone (0, 0);
601 if (!incdep_zone)
602 incdep_zone = malloc_default_zone ();
603#endif
604
605
606 /* create the mutex and two condition variables / event objects. */
607
608#ifdef HAVE_PTHREAD
609 rc = pthread_mutex_init (&incdep_mtx, NULL);
610 if (rc)
611 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
612 rc = pthread_cond_init (&incdep_cond_todo, NULL);
613 if (rc)
614 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
615 rc = pthread_cond_init (&incdep_cond_done, NULL);
616 if (rc)
617 fatal (f, _("pthread_cond_init failed: err=%d"), rc);
618
619#elif defined (WINDOWS32)
620 InitializeCriticalSection (&incdep_mtx);
621 incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
622 if (!incdep_hev_todo)
623 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
624 incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
625 if (!incdep_hev_done)
626 fatal (f, _("CreateEvent failed: err=%d"), GetLastError());
627 incdep_hev_todo_waiters = 0;
628 incdep_hev_done_waiters = 0;
629
630#elif defined (__OS2__)
631 _fmutex_create (&incdep_mtx, 0)
632 rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
633 if (rc)
634 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
635 rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
636 if (rc)
637 fatal (f, _("DosCreateEventSem failed: rc=%d"), rc);
638 incdep_hev_todo_waiters = 0;
639 incdep_hev_done_waiters = 0;
640#endif
641
642 /* create the worker threads. */
643
644 incdep_terminate = 0;
645 incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
646 if (incdep_num_threads + 1 > job_slots)
647 incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
648 for (i = 0; i < incdep_num_threads; i++)
649 {
650 alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
651 incdep_cache_allocator, (void *)i);
652 alloccache_init (&incdep_nameseq_caches[i], sizeof(struct nameseq),
653 "incdep nameseq", incdep_cache_allocator, (void *)i);
654
655#ifdef HAVE_PTHREAD
656 rc = pthread_attr_init (&attr);
657 if (rc)
658 fatal (f, _("pthread_attr_init failed: err=%d"), rc);
659 /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
660 rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
661 if (rc)
662 fatal (f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
663 rc = pthread_create(&incdep_threads[i], &attr,
664 incdep_worker_pthread, (void *)i);
665 if (rc)
666 fatal (f, _("pthread_mutex_init failed: err=%d"), rc);
667 pthread_attr_destroy (&attr);
668
669#elif defined (WINDOWS32)
670 tid = 0;
671 hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
672 (void *)i, 0, &tid);
673 if (hThread == 0 || hThread == ~(uintptr_t)0)
674 fatal (f, _("_beginthreadex failed: err=%d"), errno);
675 incdep_threads[i] = (HANDLE)hThread;
676
677#elif defined (__OS2__)
678 tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
679 if (tid <= 0)
680 fatal (f, _("_beginthread failed: err=%d"), errno);
681 incdep_threads[i] = tid;
682#endif
683 }
684
685 incdep_initialized = 1;
686}
687
688/* Flushes outstanding work and terminates the worker threads.
689 This is called from snap_deps(). */
690void
691incdep_flush_and_term (void)
692{
693 unsigned i;
694
695 if (!incdep_initialized)
696 return;
697
698 /* flush any out standing work */
699
700 incdep_flush_it (NILF);
701
702 /* tell the threads to terminate */
703
704 incdep_lock ();
705 incdep_terminate = 1;
706 incdep_signal_todo ();
707 incdep_unlock ();
708
709 /* wait for the threads to quit */
710
711 for (i = 0; i < incdep_num_threads; i++)
712 {
713 /* more later? */
714
715 alloccache_join (&dep_cache, &incdep_dep_caches[i]);
716 alloccache_join (&nameseq_cache, &incdep_nameseq_caches[i]);
717 }
718 incdep_num_threads = 0;
719
720 /* destroy the lock and condition variables / event objects. */
721
722 /* later */
723
724 incdep_initialized = 0;
725}
726
727#ifdef PARSE_IN_WORKER
728/* Flushes a strcache entry returning the actual string cache entry.
729 The input is freed! */
730static const char *
731incdep_flush_strcache_entry (const void *pv_entry)
732{
733 struct incdep_strcache_entry *entry = (struct incdep_strcache_entry *)pv_entry;
734 const char *result;
735 result = strcache_add_prehashed (entry->str, entry->length, entry->hash1, entry->hash2);
736 free (entry);
737 return result;
738}
739
740/* Flushes the recorded instructions. */
741static void
742incdep_flush_recorded_instructions (struct incdep *cur)
743{
744 struct incdep_variable_in_set *rec_vis;
745 struct incdep_variable_def *rec_vd;
746 struct incdep_recorded_files *rec_f;
747
748 /* define_variable_in_set */
749
750 rec_vis = cur->recorded_variables_in_set_head;
751 cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
752 if (rec_vis)
753 do
754 {
755 void *free_me = rec_vis;
756 unsigned int name_length = rec_vis->name_entry->length;
757 define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
758 name_length,
759 rec_vis->value,
760 rec_vis->value_length,
761 rec_vis->duplicate_value,
762 rec_vis->origin,
763 rec_vis->recursive,
764 rec_vis->set,
765 rec_vis->flocp);
766 rec_vis = rec_vis->next;
767 incdep_xfree (cur, free_me);
768 }
769 while (rec_vis);
770
771 /* do_variable_definition */
772
773 rec_vd = cur->recorded_variable_defs_head;
774 cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
775 if (rec_vd)
776 do
777 {
778 void *free_me = rec_vd;
779 do_variable_definition_2 (rec_vd->flocp,
780 incdep_flush_strcache_entry (rec_vd->name_entry),
781 rec_vd->value,
782 rec_vd->value_length,
783 0,
784 rec_vd->value,
785 rec_vd->origin,
786 rec_vd->flavor,
787 rec_vd->target_var);
788 rec_vd = rec_vd->next;
789 incdep_xfree (cur, free_me);
790 }
791 while (rec_vd);
792
793 /* record_files */
794
795 rec_f = cur->recorded_files_head;
796 cur->recorded_files_head = cur->recorded_files_tail = NULL;
797 if (rec_f)
798 do
799 {
800 void *free_me = rec_f;
801 struct dep *dep;
802 struct nameseq *filenames;
803
804 for (dep = rec_f->deps; dep; dep = dep->next)
805 dep->name = incdep_flush_strcache_entry (dep->name);
806
807 filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
808 filenames->next = 0;
809 filenames->name = incdep_flush_strcache_entry (rec_f->filename_entry);
810
811 record_files (filenames,
812 rec_f->pattern,
813 rec_f->pattern_percent,
814 rec_f->deps,
815 rec_f->cmds_started,
816 rec_f->commands,
817 rec_f->commands_idx,
818 rec_f->two_colon,
819 rec_f->flocp);
820
821 rec_f = rec_f->next;
822 incdep_xfree (cur, free_me);
823 }
824 while (rec_f);
825}
826#endif /* PARSE_IN_WORKER */
827
828/* Record / issue a warning about a misformed dep file. */
829static void
830incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
831{
832 if (cur->worker_tid == -1)
833 error (NILF, "%s(%d): %s", cur->name, line_no, msg);
834#ifdef PARSE_IN_WORKER
835 else
836 {
837 cur->err_line_no = line_no;
838 cur->err_msg = msg;
839 }
840#endif
841}
842
843/* Record / execute a strcache add. */
844static const char *
845incdep_record_strcache (struct incdep *cur, const char *str, int len)
846{
847 const char *ret;
848 if (cur->worker_tid == -1)
849 {
850 /* Make sure the string is terminated before we hand it to
851 strcache_add_len so it does have to make a temporary copy
852 of it on the stack. */
853 char ch = str[len];
854 ((char *)str)[len] = '\0';
855 ret = strcache_add_len (str, len);
856 ((char *)str)[len] = ch;
857 }
858 else
859 {
860 /* Allocate a strcache record for it, pre-hashing the string to save
861 time later on in the main thread. */
862 struct incdep_strcache_entry *entry = incdep_xmalloc (cur, sizeof (*entry) + len);
863 memcpy (entry->str, str, len);
864 entry->str[len] = '\0';
865 entry->length = len;
866 strcache_prehash_str (entry->str, &entry->hash1, &entry->hash2);
867
868 ret = (const char *)entry;
869 }
870 return ret;
871}
872
873/* Record / perform a variable definition in a set.
874 The NAME is in the string cache.
875 The VALUE is on the heap.
876 The DUPLICATE_VALUE is always 0. */
877static void
878incdep_record_variable_in_set (struct incdep *cur,
879 const char *name, unsigned int name_length,
880 const char *value,
881 unsigned int value_length,
882 int duplicate_value,
883 enum variable_origin origin,
884 int recursive,
885 struct variable_set *set,
886 const struct floc *flocp)
887{
888 assert (!duplicate_value);
889 if (cur->worker_tid == -1)
890 define_variable_in_set (name, name_length, value, value_length,
891 duplicate_value, origin, recursive, set, flocp);
892#ifdef PARSE_IN_WORKER
893 else
894 {
895 struct incdep_variable_in_set *rec = incdep_xmalloc (cur, sizeof (*rec));
896 rec->name_entry = (struct incdep_strcache_entry *)name;
897 rec->value = value;
898 rec->value_length = value_length;
899 rec->duplicate_value = duplicate_value;
900 rec->origin = origin;
901 rec->recursive = recursive;
902 rec->set = set;
903 rec->flocp = flocp;
904
905 rec->next = NULL;
906 if (cur->recorded_variables_in_set_tail)
907 cur->recorded_variables_in_set_tail->next = rec;
908 else
909 cur->recorded_variables_in_set_head = rec;
910 cur->recorded_variables_in_set_tail = rec;
911 }
912#endif
913}
914
915/* Record / perform a variable definition. The VALUE should be disposed of. */
916static void
917incdep_record_variable_def (struct incdep *cur,
918 const struct floc *flocp,
919 const char *name,
920 unsigned int name_length,
921 char *value,
922 unsigned int value_length,
923 enum variable_origin origin,
924 enum variable_flavor flavor,
925 int target_var)
926{
927 if (cur->worker_tid == -1)
928 do_variable_definition_2 (flocp, name, value, value_length, 0, value,
929 origin, flavor, target_var);
930#ifdef PARSE_IN_WORKER
931 else
932 {
933 struct incdep_variable_def *rec = incdep_xmalloc (cur, sizeof (*rec));
934 rec->flocp = flocp;
935 rec->name_entry = (struct incdep_strcache_entry *)name;
936 rec->value = value;
937 rec->value_length = value_length;
938 rec->origin = origin;
939 rec->flavor = flavor;
940 rec->target_var = target_var;
941
942 rec->next = NULL;
943 if (cur->recorded_variable_defs_tail)
944 cur->recorded_variable_defs_tail->next = rec;
945 else
946 cur->recorded_variable_defs_head = rec;
947 cur->recorded_variable_defs_tail = rec;
948 }
949#else
950 (void)name_length;
951#endif
952}
953
954/* Record files.*/
955static void
956incdep_record_files (struct incdep *cur,
957 const char *filename, const char *pattern,
958 const char *pattern_percent, struct dep *deps,
959 unsigned int cmds_started, char *commands,
960 unsigned int commands_idx, int two_colon,
961 const struct floc *flocp)
962{
963 if (cur->worker_tid == -1)
964 {
965 struct nameseq *filenames = (struct nameseq *) alloccache_alloc (&nameseq_cache);
966 filenames->next = 0;
967 filenames->name = filename;
968 record_files (filenames, pattern, pattern_percent, deps, cmds_started,
969 commands, commands_idx, two_colon, flocp);
970 }
971#ifdef PARSE_IN_WORKER
972 else
973 {
974 struct incdep_recorded_files *rec = incdep_xmalloc (cur, sizeof (*rec));
975
976 rec->filename_entry = (struct incdep_strcache_entry *)filename;
977 rec->pattern = pattern;
978 rec->pattern_percent = pattern_percent;
979 rec->deps = deps;
980 rec->cmds_started = cmds_started;
981 rec->commands = commands;
982 rec->commands_idx = commands_idx;
983 rec->two_colon = two_colon;
984 rec->flocp = flocp;
985
986 rec->next = NULL;
987 if (cur->recorded_files_tail)
988 cur->recorded_files_tail->next = rec;
989 else
990 cur->recorded_files_head = rec;
991 cur->recorded_files_tail = rec;
992 }
993#endif
994}
995
996
997/* no nonsense dependency file including.
998
999 Because nobody wants bogus dependency files to break their incremental
1000 builds with hard to comprehend error messages, this function does not
1001 use the normal eval routine but does all the parsing itself. This isn't,
1002 as much work as it sounds, because the necessary feature set is very
1003 limited.
1004
1005 eval_include_dep_file groks:
1006
1007 define var
1008 endef
1009
1010 var [|:|?|>]= value [\]
1011
1012 [\]
1013 file: [deps] [\]
1014
1015 */
1016static void
1017eval_include_dep_file (struct incdep *curdep, struct floc *f)
1018{
1019 unsigned line_no = 1;
1020 const char *file_end = curdep->file_end;
1021 const char *cur = curdep->file_base;
1022 const char *endp;
1023
1024 /* if no file data, just return immediately. */
1025 if (!cur)
1026 return;
1027
1028 /* now parse the file. */
1029 while (cur < file_end)
1030 {
1031 /* skip empty lines */
1032 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1033 ++cur;
1034 if (cur >= file_end)
1035 break;
1036 if (*cur == '#')
1037 {
1038 cur = memchr (cur, '\n', file_end - cur);
1039 if (!cur)
1040 break;
1041 }
1042 if (*cur == '\\')
1043 {
1044 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1045 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1046 : (file_end - cur == 1) ? 1 : 0;
1047 if (eol_len)
1048 {
1049 cur += eol_len;
1050 line_no++;
1051 continue;
1052 }
1053 }
1054 if (*cur == '\n')
1055 {
1056 cur++;
1057 line_no++;
1058 continue;
1059 }
1060
1061 /* define var
1062 ...
1063 endef */
1064 if (strneq (cur, "define ", 7))
1065 {
1066 const char *var;
1067 unsigned var_len;
1068 const char *value_start;
1069 const char *value_end;
1070 char *value;
1071 unsigned value_len;
1072 int found_endef = 0;
1073
1074 /* extract the variable name. */
1075 cur += 7;
1076 while (isblank ((unsigned char)*cur))
1077 ++cur;
1078 value_start = endp = memchr (cur, '\n', file_end - cur);
1079 if (!endp)
1080 endp = cur;
1081 while (endp > cur && isspace ((unsigned char)endp[-1]))
1082 --endp;
1083 var_len = endp - cur;
1084 if (!var_len)
1085 {
1086 incdep_warn (curdep, line_no, "bogus define statement.");
1087 break;
1088 }
1089 var = incdep_record_strcache (curdep, cur, var_len);
1090
1091 /* find the end of the variable. */
1092 cur = value_end = value_start = value_start + 1;
1093 ++line_no;
1094 while (cur < file_end)
1095 {
1096 /* check for endef, don't bother with skipping leading spaces. */
1097 if ( file_end - cur >= 5
1098 && strneq (cur, "endef", 5))
1099 {
1100 endp = cur + 5;
1101 while (endp < file_end && isspace ((unsigned char)*endp) && *endp != '\n')
1102 endp++;
1103 if (endp >= file_end || *endp == '\n')
1104 {
1105 found_endef = 1;
1106 cur = endp >= file_end ? file_end : endp + 1;
1107 break;
1108 }
1109 }
1110
1111 /* skip a line ahead. */
1112 cur = value_end = memchr (cur, '\n', file_end - cur);
1113 if (cur != NULL)
1114 ++cur;
1115 else
1116 cur = value_end = file_end;
1117 ++line_no;
1118 }
1119
1120 if (!found_endef)
1121 {
1122 incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
1123 break;
1124 }
1125 value_len = value_end - value_start;
1126 if (memchr (value_start, '\0', value_len))
1127 {
1128 incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
1129 break;
1130 }
1131
1132 /* make a copy of the value, converting \r\n to \n, and define it. */
1133 value = incdep_xmalloc (curdep, value_len + 1);
1134 endp = memchr (value_start, '\r', value_len);
1135 if (endp)
1136 {
1137 const char *src = value_start;
1138 char *dst = value;
1139 for (;;)
1140 {
1141 size_t len = endp - src;
1142 memcpy (dst, src, len);
1143 dst += len;
1144 src = endp;
1145 if (src + 1 < file_end && src[1] == '\n')
1146 src++; /* skip the '\r' */
1147 if (src >= value_end)
1148 break;
1149 endp = memchr (endp + 1, '\r', src - value_end);
1150 if (!endp)
1151 endp = value_end;
1152 }
1153 value_len = dst - value;
1154 }
1155 else
1156 memcpy (value, value_start, value_len);
1157 value [value_len] = '\0';
1158
1159 incdep_record_variable_in_set (curdep,
1160 var, var_len, value, value_len,
1161 0 /* don't duplicate */, o_file,
1162 0 /* defines are recursive but this is faster */,
1163 NULL /* global set */, f);
1164 }
1165
1166 /* file: deps
1167 OR
1168 variable [:]= value */
1169 else
1170 {
1171 const char *colonp;
1172 const char *equalp;
1173
1174 /* Look for a colon and an equal sign, optimize for colon.
1175 Only one file is support and the colon / equal must be on
1176 the same line. */
1177 colonp = memchr (cur, ':', file_end - cur);
1178#ifdef HAVE_DOS_PATHS
1179 while ( colonp
1180 && colonp + 1 < file_end
1181 && (colonp[1] == '/' || colonp[1] == '\\')
1182 && colonp > cur
1183 && isalpha ((unsigned char)colonp[-1])
1184 && ( colonp == cur + 1
1185 || strchr (" \t(", colonp[-2]) != 0))
1186 colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
1187#endif
1188 endp = NULL;
1189 if ( !colonp
1190 || (endp = memchr (cur, '\n', colonp - cur)))
1191 {
1192 colonp = NULL;
1193 equalp = memchr (cur, '=', (endp ? endp : file_end) - cur);
1194 if ( !equalp
1195 || (!endp && memchr (cur, '\n', equalp - cur)))
1196 {
1197 incdep_warn (curdep, line_no, "no colon.");
1198 break;
1199 }
1200 }
1201 else
1202 equalp = memchr (cur, '=', (colonp + 2 <= file_end
1203 ? colonp + 2 : file_end) - cur);
1204 if (equalp)
1205 {
1206 /* An assignment of some sort. */
1207 const char *var;
1208 unsigned var_len;
1209 const char *value_start;
1210 const char *value_end;
1211 char *value;
1212 unsigned value_len;
1213 unsigned multi_line = 0;
1214 enum variable_flavor flavor;
1215
1216 /* figure the flavor first. */
1217 flavor = f_recursive;
1218 if (equalp > cur)
1219 {
1220 if (equalp[-1] == ':')
1221 flavor = f_simple;
1222 else if (equalp[-1] == '?')
1223 flavor = f_conditional;
1224 else if (equalp[-1] == '+')
1225 flavor = f_append;
1226 else if (equalp[-1] == '>')
1227 flavor = f_prepend;
1228 }
1229
1230 /* extract the variable name. */
1231 endp = flavor == f_recursive ? equalp : equalp - 1;
1232 while (endp > cur && isblank ((unsigned char)endp[-1]))
1233 --endp;
1234 var_len = endp - cur;
1235 if (!var_len)
1236 {
1237 incdep_warn (curdep, line_no, "empty variable. (includedep)");
1238 break;
1239 }
1240 if ( memchr (cur, '$', var_len)
1241 || memchr (cur, ' ', var_len)
1242 || memchr (cur, '\t', var_len))
1243 {
1244 incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
1245 break;
1246 }
1247 var = incdep_record_strcache (curdep, cur, var_len);
1248
1249 /* find the start of the value. */
1250 cur = equalp + 1;
1251 while (cur < file_end && isblank ((unsigned char)*cur))
1252 cur++;
1253 value_start = cur;
1254
1255 /* find the end of the value / line (this isn't 101% correct). */
1256 value_end = cur;
1257 while (cur < file_end)
1258 {
1259 endp = value_end = memchr (cur, '\n', file_end - cur);
1260 if (!value_end)
1261 value_end = file_end;
1262 if (value_end - 1 >= cur && value_end[-1] == '\r')
1263 --value_end;
1264 if (value_end - 1 < cur || value_end[-1] != '\\')
1265 {
1266 cur = endp ? endp + 1 : file_end;
1267 break;
1268 }
1269 --value_end;
1270 if (value_end - 1 >= cur && value_end[-1] == '\\')
1271 {
1272 incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
1273 cur = NULL;
1274 break;
1275 }
1276 if (!endp)
1277 {
1278 cur = file_end;
1279 break;
1280 }
1281
1282 cur = endp + 1;
1283 ++multi_line;
1284 ++line_no;
1285 }
1286 if (!cur)
1287 break;
1288 ++line_no;
1289
1290 /* make a copy of the value, converting \r\n to \n, and define it. */
1291 value_len = value_end - value_start;
1292 value = incdep_xmalloc (curdep, value_len + 1);
1293 if (!multi_line)
1294 memcpy (value, value_start, value_len);
1295 else
1296 {
1297 /* unescape it */
1298 const char *src = value_start;
1299 char *dst = value;
1300 while (src < value_end)
1301 {
1302 const char *nextp;
1303
1304 endp = memchr (src, '\n', value_end - src);
1305 if (!endp)
1306 nextp = endp = value_end;
1307 else
1308 nextp = endp + 1;
1309 if (endp > src && endp[-1] == '\r')
1310 --endp;
1311 if (endp > src && endp[-1] == '\\')
1312 --endp;
1313
1314 if (src != value_start)
1315 *dst++ = ' ';
1316 memcpy (dst, src, endp - src);
1317 dst += endp - src;
1318 src = nextp;
1319 }
1320 value_len = dst - value;
1321 }
1322 value [value_len] = '\0';
1323
1324 /* do the definition */
1325 if (flavor == f_recursive
1326 || ( flavor == f_simple
1327 && !memchr (value, '$', value_len)))
1328 incdep_record_variable_in_set (curdep,
1329 var, var_len, value, value_len,
1330 0 /* don't duplicate */, o_file,
1331 flavor == f_recursive /* recursive */,
1332 NULL /* global set */, f);
1333 else
1334 incdep_record_variable_def (curdep,
1335 f, var, var_len, value, value_len,
1336 o_file, flavor, 0 /* not target var */);
1337 }
1338 else
1339 {
1340 /* file: dependencies */
1341
1342 const char *filename;
1343 struct dep *deps = 0;
1344 struct dep **nextdep = &deps;
1345 struct dep *dep;
1346
1347 /* extract the filename, ASSUME a single one. */
1348 endp = colonp;
1349 while (endp > cur && isblank ((unsigned char)endp[-1]))
1350 --endp;
1351 if (cur == endp)
1352 {
1353 incdep_warn (curdep, line_no, "empty filename.");
1354 break;
1355 }
1356 if ( memchr (cur, '$', endp - cur)
1357 || memchr (cur, ' ', endp - cur)
1358 || memchr (cur, '\t', endp - cur))
1359 {
1360 incdep_warn (curdep, line_no, "multiple / fancy file name. (includedep)");
1361 break;
1362 }
1363 filename = incdep_record_strcache (curdep, cur, endp - cur);
1364
1365 /* parse any dependencies. */
1366 cur = colonp + 1;
1367 while (cur < file_end)
1368 {
1369 /* skip blanks and count lines. */
1370 while (cur < file_end && isspace ((unsigned char)*cur) && *cur != '\n')
1371 ++cur;
1372 if (cur >= file_end)
1373 break;
1374 if (*cur == '\n')
1375 {
1376 cur++;
1377 line_no++;
1378 break;
1379 }
1380
1381 /* continuation + eol? */
1382 if (*cur == '\\')
1383 {
1384 unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
1385 : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
1386 : (file_end - cur == 1) ? 1 : 0;
1387 if (eol_len)
1388 {
1389 cur += eol_len;
1390 line_no++;
1391 continue;
1392 }
1393 }
1394
1395 /* find the end of the filename */
1396 endp = cur;
1397 while (endp < file_end && !isspace ((unsigned char)*endp))
1398 ++endp;
1399
1400 /* add it to the list. */
1401 *nextdep = dep = incdep_alloc_dep (curdep);
1402 dep->name = incdep_record_strcache (curdep, cur, endp - cur);
1403 dep->includedep = 1;
1404 nextdep = &dep->next;
1405
1406 cur = endp;
1407 }
1408
1409 /* enter the file with its dependencies. */
1410 incdep_record_files (curdep,
1411 filename, NULL, NULL, deps, 0, NULL, 0, 0, f);
1412 }
1413 }
1414 }
1415
1416 /* free the file data */
1417 incdep_xfree (curdep, curdep->file_base);
1418 curdep->file_base = curdep->file_end = NULL;
1419}
1420
1421/* Flushes the incdep todo and done lists. */
1422static void
1423incdep_flush_it (struct floc *f)
1424{
1425 incdep_lock ();
1426 for (;;)
1427 {
1428 struct incdep *cur = incdep_head_done;
1429
1430 /* if the done list is empty, grab a todo list entry. */
1431 if (!cur && incdep_head_todo)
1432 {
1433 cur = incdep_head_todo;
1434 if (cur->next)
1435 incdep_head_todo = cur->next;
1436 else
1437 incdep_head_todo = incdep_tail_todo = NULL;
1438 incdep_unlock ();
1439
1440 incdep_read_file (cur, f);
1441 eval_include_dep_file (cur, f);
1442 incdep_freeit (cur);
1443
1444 incdep_lock ();
1445 continue;
1446 }
1447
1448 /* if the todo list and done list are empty we're either done
1449 or will have to wait for the thread(s) to finish. */
1450 if (!cur && !incdep_num_reading)
1451 break; /* done */
1452 if (!cur)
1453 {
1454 while (!incdep_head_done)
1455 incdep_wait_done ();
1456 cur = incdep_head_done;
1457 }
1458
1459 /* we grab the entire done list and work thru it. */
1460 incdep_head_done = incdep_tail_done = NULL;
1461 incdep_unlock ();
1462
1463 while (cur)
1464 {
1465 struct incdep *next = cur->next;
1466#ifdef PARSE_IN_WORKER
1467 incdep_flush_recorded_instructions (cur);
1468#else
1469 eval_include_dep_file (cur, f);
1470#endif
1471 incdep_freeit (cur);
1472 cur = next;
1473 }
1474
1475 incdep_lock ();
1476 } /* outer loop */
1477 incdep_unlock ();
1478}
1479
1480
1481/* splits up a list of file names and feeds it to eval_include_dep_file,
1482 employing threads to try speed up the file reading. */
1483void
1484eval_include_dep (const char *names, struct floc *f, enum incdep_op op)
1485{
1486 struct incdep *head = 0;
1487 struct incdep *tail = 0;
1488 struct incdep *cur;
1489 const char *names_iterator = names;
1490 const char *name;
1491 unsigned int name_len;
1492
1493 /* loop through NAMES, creating a todo list out of them. */
1494
1495 while ((name = find_next_token (&names_iterator, &name_len)) != 0)
1496 {
1497 cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
1498 cur->file_base = cur->file_end = NULL;
1499 memcpy (cur->name, name, name_len);
1500 cur->name[name_len] = '\0';
1501 cur->worker_tid = -1;
1502#ifdef PARSE_IN_WORKER
1503 cur->err_line_no = 0;
1504 cur->err_msg = NULL;
1505 cur->recorded_variables_in_set_head = NULL;
1506 cur->recorded_variables_in_set_tail = NULL;
1507 cur->recorded_variable_defs_head = NULL;
1508 cur->recorded_variable_defs_tail = NULL;
1509 cur->recorded_files_head = NULL;
1510 cur->recorded_files_tail = NULL;
1511#endif
1512
1513 cur->next = NULL;
1514 if (tail)
1515 tail->next = cur;
1516 else
1517 head = cur;
1518 tail = cur;
1519 }
1520
1521#ifdef ELECTRIC_HEAP
1522 if (1)
1523#else
1524 if (op == incdep_read_it)
1525#endif
1526 {
1527 /* work our way thru the files directly */
1528
1529 cur = head;
1530 while (cur)
1531 {
1532 struct incdep *next = cur->next;
1533 incdep_read_file (cur, f);
1534 eval_include_dep_file (cur, f);
1535 incdep_freeit (cur);
1536 cur = next;
1537 }
1538 }
1539 else
1540 {
1541 /* initialize the worker threads and related stuff the first time around. */
1542
1543 if (!incdep_initialized)
1544 incdep_init (f);
1545
1546 /* queue the files and notify the worker threads. */
1547
1548 incdep_lock ();
1549
1550 if (incdep_tail_todo)
1551 incdep_tail_todo->next = head;
1552 else
1553 incdep_head_todo = head;
1554 incdep_tail_todo = tail;
1555
1556 incdep_signal_todo ();
1557 incdep_unlock ();
1558
1559 /* flush the todo queue if we're requested to do so. */
1560
1561 if (op == incdep_flush)
1562 incdep_flush_it (f);
1563 }
1564}
1565
1566#endif /* CONFIG_WITH_INCLUDEDEP */
1567
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette