Changing the way fetching is handled.
[bitbake.git] / bin / oebuild
1 #!/usr/bin/python
2
3 import sys,os,string
4 sys.path.append(os.path.join(sys.prefix,'share/oe'))
5 from oe import emit_env
6 from oe import *
7
8
9 #######################################################################
10 class FuncFailed(Exception):
11         """Executed function failed"""
12
13 class TaskBase(event.Event):
14         """Base class for task events"""
15
16         def __init__(self, t, d = {}):
17                 self.task = t
18                 self.data = d
19
20         def getTask(self):
21                 return self._task
22
23         def setTask(self, task):
24                 self._task = task
25
26         task = property(getTask, setTask, None, "task property")
27
28         def getData(self):
29                 return self._data
30
31         def setData(self, data):
32                 self._data = data
33
34         data = property(getData, setData, None, "data property")
35
36 class TaskStarted(TaskBase):
37         """Task execution started"""
38         
39 class TaskSucceeded(TaskBase):
40         """Task execution succeeded"""
41
42 class TaskFailed(TaskBase):
43         """Task execution failed"""
44
45 def exec_shell_func(func, d=None, myfatal=fatal):
46         """This executes any shell function stored in env[func]. Returns
47         true if execution was successful.
48
49         For this, it creates a bash shell script in the tmp dectory, writes the local
50         environment in env[] into it and finally executes this. The output of the
51         shell will end in a log file and stdout.
52         """
53
54         if tasks.has_key(func) and tasks[func].has_key('deps'):
55                 deps = tasks[func]['deps']
56         else:
57                 deps = None
58
59         if tasks.has_key(func) and tasks[func].has_key('check'):
60                 checkfunc = tasks[func]['check']
61         else:
62                 checkfunc = 'stamp_is_current'
63
64         if globals()[checkfunc](func, deps):
65                 return
66
67         global logfile
68         mkdirhier(getenv('T'))
69         logfile = getenv('T') + '/log.' + func + '.' + str(os.getpid())
70         runfile = getenv('T') + '/run.' + func + '.' + str(os.getpid())
71
72         f = open(runfile, "w")
73         f.write("#!/bin/bash\n")
74         if env.has_key('OEDEBUG'): f.write("set -x\n")
75         if env.has_key('OEPATH'):
76                 for s in expand(env['OEPATH']).split(":"):
77                         f.write("if test -f %s/build/oebuild.sh; then source %s/build/oebuild.sh; fi\n" % (s,s));
78         emit_env(f)
79
80         if d:  f.write("cd %s\n" % env[d])
81         if func: f.write(func +"\n")
82         f.close()
83         os.chmod(runfile, 0775)
84         if not func:
85                 error("Function not specified")
86                 raise FuncFailed()
87         event.fire(TaskStarted(func, env))
88         ret = os.system("bash -c 'source %s' 2>&1 | tee %s; exit $PIPESTATUS" % (runfile, logfile))
89         if ret==0:
90                 if not env.has_key('OEDEBUG'): 
91                         os.remove(runfile)
92                         os.remove(logfile)
93                 event.fire(TaskSucceeded(func, env))
94                 mkstamp(func)
95                 return
96         else:
97                 error("'%s'() failed" % func);
98                 error("more info in: %s" % logfile);
99                 event.fire(TaskFailed(func, env))
100                 raise FuncFailed()
101
102
103 def exec_python_func(func, dirname = None, myfatal = fatal):
104         if tasks.has_key(func) and tasks[func].has_key('deps'):
105                 deps = tasks[func]['deps']
106         else:
107                 deps = None
108
109         if tasks.has_key(func) and tasks[func].has_key('check'):
110                 checkfunc = tasks[func]['check']
111         else:
112                 checkfunc = 'stamp_is_current'
113
114         if globals()[checkfunc](func, deps):
115                 return
116
117         if not env.has_key(dirname):
118                 warn("dir variable %s does not exist, cannot chdir" % dirname)
119                 return
120
121         event.fire(TaskStarted(func, env))
122
123         dir = env[dirname]
124
125         if not os.path.isdir(dir):
126                 debug(1, "%s does not exist, creating it." % dir)
127                 mkdirhier(dir)
128
129         os.chdir(dir)
130         try:
131                 comp = compile(getenv(func),func,'exec')
132                 exec(comp)
133         except Exception, e:
134                 (type, value, traceback) = sys.exc_info()
135                 error("Python function %s failed: %s" % (func, value))
136                 event.fire(TaskFailed(func, env))
137                 raise FuncFailed(value)
138         event.fire(TaskSucceeded(func, env))
139         mkstamp(func)
140
141
142 #######################################################################
143 __success = []
144
145 def exec_task(task, dir, deptasks = []):
146         """Executes a given task, handling the stamps appropriately"""
147
148         if task in __success:
149                 return
150
151         if not deptasks:
152                 try:
153                         deptasks = tasks[task]["deps"]
154                 except KeyError:
155                         pass
156
157         for dep in deptasks:
158                 try:
159                         depfuncname = tasks[dep]["func"]
160                         depfunc = globals()[depfuncname]
161                         depfunc()
162                         __success.append(dep)
163                 except KeyError:
164                         debug(2, "%s: invalid 'func' for task %s, running exec_task directly." % (task, dep))
165                         if tasks.has_key(dep) and tasks[dep].has_key("deps"):
166                                 exec_task(dep, dir, tasks[dep]["deps"])
167                         else:
168                                 exec_task(dep, dir)
169
170         if not tasks.has_key(task):
171                 if not envflags.has_key(task) and not env.has_key(task):
172                         error("unsupported task: %s" % task)
173                         raise FuncFailed()
174
175         debug(1, "Executing task '%s'" % task)
176
177         if envflags.has_key(task) and envflags[task].has_key("python") and envflags[task]["python"] is not None:
178                 # execute this function as a block of python code, not shell
179                 exec_python_func(task, dir)
180         else:
181                 exec_shell_func(task, dir)
182         __success.append(task)
183
184 def mkstamp(task):
185         """Creates/updates a stamp for a given task"""
186         mkdirhier(expand('${TMPDIR}/stamps'));
187         open(getenv('STAMP')+"."+task, "w+")
188
189 def stamp_is_current(task, deptasks=""):
190         """Check status of a given task's stamp. returns False if it is not current and needs updating."""
191         stampfile = getenv('STAMP') + "." + task
192         if not os.access(stampfile, os.F_OK):
193                 return False
194
195         import stat
196         tasktime = os.stat(stampfile)[stat.ST_MTIME]
197         if deptasks is None:
198                 return True
199
200         for dep in deptasks:
201                 depfile = getenv('STAMP') + "." + dep
202                 if not os.access(depfile, os.F_OK):
203                         return True
204                 deptime = os.stat(depfile)[stat.ST_MTIME]
205                 if deptime > tasktime:
206                         return False
207
208         return True
209
210 def check_md5(task, deptasks):
211         md5s = []
212         if env.has_key(task):
213                 # this is an environment task.
214                 md5s.append(env[task])
215         if tasks.has_key(task) and tasks[task].has_key('md5check'):
216                 for dep in tasks[task]['md5check']:
217                         if env.has_key(dep):
218                                 md5s.append(env[dep])
219
220         import md5
221         TaskMD5 = md5.new()
222         for m in md5s:
223                 TaskMD5.update(m)
224
225         def write_md5(task, digest):
226                 path = os.path.join(os.path.dirname(getenv('STAMP')),"md5."+getenv('PF')+'.'+task)
227                 try:
228                         stamp = open(path, "wb+")
229                 except Exception:
230                         return None
231
232                 stamp.write(digest)
233                 stamp.close()
234
235         def read_md5(task):
236                 path = os.path.join(os.path.dirname(getenv('STAMP')),"md5."+getenv('PF')+'.'+task)
237                 try:
238                         stamp = open(path, "rb+")
239                 except Exception:
240                         return None
241
242                 digest = stamp.read()
243                 stamp.close()
244                 return digest
245         # follow dependency path, calling 'undo' functions to cleanup
246         def followdown(item, callback):
247                 try:
248                         deps = tasks[item]['deps']
249                 except KeyError:
250                         deps = None
251                 if deps is not None:
252                         for p in deps:
253                                 ret = followdown(p, callback)
254                 return callback(item)
255
256         def followup(item, callback):
257                 deps = []
258                 for task in tasks:
259                         try:
260                                 if item in tasks[task]['deps']:
261                                         deps.append(task)
262                         except KeyError:
263                                 continue
264
265                 if deps is not None:
266                         for p in deps:
267                                 ret = followup(p, callback)
268                 return callback(item)
269
270         def undo(task):
271                 undo = None
272                 undodeps = None
273                 try:
274                         undo = tasks[task]['undo']
275                         undodeps = tasks[undo]['deps']
276                 except KeyError:
277                         pass
278                 if not undo: return
279                 try:
280                         depfuncname = tasks[undo]["func"]
281                         depfunc = globals()[depfuncname]
282                         depfunc()
283                 except KeyError:
284                         if undodeps:
285                                 exec_task(undo, None, undodeps)
286                         else:
287                                 exec_task(undo, None)
288
289
290         old = read_md5(task)
291         new = TaskMD5.digest()
292         if old is None or old != new:
293                 # md5sum has changed
294                 # remove the stamp so that next time it rebuilds regardless
295                 try:
296                         os.remove(getenv('STAMP')+"."+task)
297                 except OSError, IOError:
298                         pass
299
300                 followup(task, undo)
301
302                 # update md5
303                 write_md5(task, new)
304                 return 0
305
306         return stamp_is_current(task, deptasks)
307
308 #TODO: rmstamp
309
310
311 #######################################################################
312
313 def do_clean():
314         """clear the build and temp directories"""
315
316         note("Executing task 'clean'")
317
318         dir = expand("${TMPDIR}/${CATEGORY}/${PF}")
319         if dir == '//': fatal("wrong DATADIR")
320         note("removing " + dir)
321         os.system('rm -rf ' + dir)
322
323         dir = getenv('STAMP')+'.*'
324         note("removing " + dir)
325         os.system('rm -f '+ dir)
326
327
328 #######################################################################
329
330 def do_mrproper():
331         """clear downloaded sources, build and temp directories"""
332
333         note("Executing task 'mrproper'")
334
335         dir = getenv("DL_DIR")
336         if dir == '/': fatal("wrong DATADIR");
337         debug(2, "removing " + dir)
338         os.system('rm -rf ' + dir)
339         do_clean()
340
341
342 #######################################################################
343
344 def do_fetch():
345         """download needed sources"""
346
347         event.fire(TaskStarted('do_fetch', env))
348
349         mkdirhier(getenv("DL_DIR"))     # create file download directory
350
351         if env.has_key('SRC_URI'):
352                 try:
353                         fetch.init(getenv('SRC_URI').split())
354                 except fetch.NoMethodError:
355                         (type, value, traceback) = sys.exc_info()
356                         error("No method: %s" % value)
357                         event.fire(TaskFailed('do_fetch', env))
358                         raise FuncFailed()
359
360                 try:
361                         fetch.go()
362                 except fetch.MissingParameterError:
363                         (type, value, traceback) = sys.exc_info()
364                         error("Missing parameters: %s" % value)
365                         event.fire(TaskFailed('do_fetch', env))
366                         raise FuncFailed()
367                 except fetch.FetchError:
368                         (type, value, traceback) = sys.exc_info()
369                         error("Fetch failed: %s" % value)
370                         event.fire(TaskFailed('do_fetch', env))
371                         raise FuncFailed()
372         else:
373                 note("No SRC_URI variable, nothing to be done")
374         event.fire(TaskSucceeded('do_fetch', env))
375
376
377 #######################################################################
378
379 def do_unpack():
380         """unpack downloaded sources, patch sources if needed"""
381
382         mkdirhier(env["WORKDIR"])       # create workdir
383         
384         exec_task('do_unpack', 'WORKDIR')
385
386
387 #######################################################################
388
389 def do_compile():
390         """compile extracted sources"""
391
392         mkdirhier(env["S"])             # create sourcedir if not yet existing
393         exec_task('do_compile', 'S')
394
395
396 #######################################################################
397
398 def do_stage():
399         """install needed files to compile other packages into staging directory"""
400
401         mkdirhier(getenv("S"))          # create sourcedir if not yet existing
402         mkdirhier(getenv("STAGING_LIBDIR"))
403         mkdirhier(getenv("STAGING_BINDIR"))
404         mkdirhier(getenv("STAGING_DIR") + "/build/include")
405         mkdirhier(getenv("STAGING_DIR") + "/target/include")
406         exec_task('do_stage', 'S')
407                 
408
409 #######################################################################
410
411 def do_install():
412         """install compiled files into image directory"""
413
414         mkdirhier(env["S"])             # create sourcedir if not yet existing
415         exec_task('do_install', 'S')
416                 
417
418 #######################################################################
419
420 def do_build():
421         """fetch, extract, compile, stage and install files"""
422         
423         do_fetch()
424         print_orphan_env()
425         print_missing_env()
426         #emit_env()
427         do_install()
428
429
430
431 #######################################################################
432
433 def do_test():
434         """internal"""
435
436         ret = exec_shell_func(None)
437
438
439
440 #######################################################################
441
442 def do_showenv():
443         """internal
444
445         Just for testing purposes. Displays all environment vars that
446         are NOT automatically generated by oebuild, but are in the config
447         file.
448
449         Might give 'false positives' if some environment variable has not
450         yet been documented in py's envflags{}.
451         """
452
453         keys = env.keys()
454         keys.sort()
455         for s in keys:
456                 #if envflags.has_key(s): continue
457                 
458                 var = env[s]
459                 var = expand(var)
460
461                 if s == s.lower():
462                         print s + '() {\n' + var + '}'
463                 else:
464                         print s+'=' + var
465
466         print_missing_env()
467         print_orphan_env()
468
469
470 #######################################################################
471
472 def usage(errorlevel=0, txt=''):
473         global tasks
474         if txt:
475                 print
476                 print txt
477
478         funcs = []
479         for i in tasks.keys():
480                 if tasks[i].has_key("func"):
481                         funcs.append(tasks[i]["func"])
482         funcs.sort()
483
484         print "\noebuild <command> <somebuildfile.oe>\n"
485         print "Valid commands are:"
486         for s in funcs:
487                 if not globals().has_key(s):
488                         continue
489                 doc = getattr(globals()[s], '__doc__')
490                 if doc.startswith('internal'): continue
491                 print "     %-15s %s" % (s[3:], doc)
492         
493         sys.exit(errorlevel)
494
495 tasks = {
496         "do_clean" : { "func" : "do_clean",
497                      },
498         "do_mrproper" : { "func" : "do_mrproper",
499                          },
500         "do_test" : { "func" : "do_test",
501                     },
502         "do_showenv" : { "func" : "do_showenv",
503                        },
504         "do_fetch" : { "func" : "do_fetch",
505                       "check" : "check_md5",
506                       "md5check" : ["SRC_URI"],
507                       "deps" : [ ],
508                      },
509         "do_unpack" : { "func" : "do_unpack",
510                       "check" : "check_md5",
511                       "md5check" : ["A"],
512                       "undo" : "do_clean",
513                       "deps" : [ 'do_fetch' ],
514                       },
515         "do_compile" : { "func" : "do_compile",
516                        "check" : "check_md5",
517                        "deps" : [ 'do_unpack' ],
518                        },
519         "do_patch" : { "undo" : "do_clean",
520                      "check" : "check_md5",
521                      "md5check" : ["A"],
522                      },
523         "do_stage" : { "func" : "do_stage",
524                      "check" : "check_md5",
525                      "deps" : [ 'do_compile' ],
526                      },
527         "do_install" : { "func" : "do_install",
528                        "check" : "check_md5",
529                        "deps" : [ 'do_stage' ],
530                        },
531         "do_package" : { "func" : "do_package",
532                        "check" : "check_md5",
533                        "deps" : [ 'do_install' ],
534                        },
535         "do_build" : { "func" : "do_build",
536                      "deps" : [ 'do_package' ],
537                      },
538 }
539
540 if len(sys.argv) < 2:
541         usage(1)
542 if sys.argv[1] in ('help', 'usage'):
543         usage(0);
544 if len(sys.argv) < 3:
545         usage(1)
546         fatal("try 'oebuild <command> <somebuildfile.oe>'")
547
548 inherit_os_env(1)
549
550 __oepath_found_it__ = 0
551
552 for s in getenv('OEPATH').split(":"):
553         if os.access(os.path.join(s,'conf/oe.conf'), os.R_OK):
554                 __oepath_found_it__ = 1
555                 try:
556                         read_config(os.path.join(s,'conf/oe.conf'))     # Read configuration
557                 except IOError:
558                         pass
559
560 if __oepath_found_it__ == 0:
561         fatal("error locating conf/oe.conf")
562
563 set_automatic_vars(sys.argv[2])                 # Deduce per-package environment variables
564 try:
565         read_oe(sys.argv[2])            # Read package configuration
566 except IOError:
567         fatal("error accessing build file")
568 set_additional_vars()                           # set rest of vars, e.g. ${A}
569 update_env()                                    # Environment modification, e.g. local overrides
570
571 # check for functions to insert into the task stack
572 for var in env.keys():
573         if not envflags.has_key(var):
574                 continue
575         if envflags[var].has_key("handler") and env.has_key(var):
576                 event.register(env[var])
577                 continue
578         if not envflags[var].has_key("task"):
579                 continue
580
581         taskvar = var
582         tasks[taskvar] = {}
583         if envflags[var].has_key("func"):
584                 tasks[taskvar]["func"] = envflags[var]["func"]
585         else:
586                 tasks[taskvar]["func"] = "INVALID"
587         if envflags[var].has_key("deps"):
588                 # other tasks that we depend on
589                 tasks[taskvar]["deps"] = envflags[var]["deps"] 
590         if envflags[var].has_key("postdeps"):
591                 # other tasks that depend on this one
592                 postdeps = envflags[var]["postdeps"]
593                 for t in tasks.keys():
594                         for i in postdeps:
595                                 if t == i:
596                                         if not tasks[t].has_key("deps"):
597                                                 tasks[t]["deps"] = []
598                                         tasks[t]["deps"].append(taskvar)
599                                         break
600                         
601         if envflags[var].has_key("python"):
602                 tasks[taskvar]["python"] = "1"
603
604 for s in sys.argv[1].split(','):
605         try:
606                 try:
607                         funcname = tasks["do_" + s]["func"]
608                         func = globals()[funcname]
609                         func()
610                 except KeyError:
611                         exec_task("do_" + s, "S")
612         except FuncFailed:
613                 fatal("task \"%s\" failed" % s)
614
615         note("task \"%s\" completed" % s)
616 #               INVALID()
617 #               usage(1, "'%s' is not valid command" % s)
618
619 sys.exit(0)