123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554 |
- #! /usr/bin/python2
- import os.path
- import sys
- import shlex
- import re
- import subprocess
- import shutil
- import pickle
- import multiprocessing
- def find_pound_include (line, use_outside, use_slash):
- inc = re.findall (ur"^\s*#\s*include\s*\"(.+?)\"", line)
- if len(inc) == 1:
- nm = inc[0]
- if use_outside or os.path.exists (nm):
- if use_slash or '/' not in nm:
- return nm
- return ""
- def find_system_include (line):
- inc = re.findall (ur"^\s*#\s*include\s*<(.+?)>", line)
- if len(inc) == 1:
- return inc[0]
- return ""
-
- def find_pound_define (line):
- inc = re.findall (ur"^\s*#\s*define ([A-Za-z0-9_]+)", line)
- if len(inc) != 0:
- if len(inc) > 1:
- print "What? more than 1 match in #define??"
- print inc
- sys.exit(5)
- return inc[0];
- return ""
- def is_pound_if (line):
- inc = re.findall ("^\s*#\s*if\s", line)
- if not inc:
- inc = re.findall ("^\s*#\s*if[n]?def\s", line)
- if inc:
- return True
- return False
- def is_pound_endif (line):
- inc = re.findall ("^\s*#\s*endif", line)
- if inc:
- return True
- return False
- def find_pound_if (line):
- inc = re.findall (ur"^\s*#\s*if\s+(.*)", line)
- if len(inc) == 0:
- inc = re.findall (ur"^\s*#\s*elif\s+(.*)", line)
- if len(inc) > 0:
- inc2 = re.findall (ur"defined\s*\((.+?)\)", inc[0])
- inc3 = re.findall (ur"defined\s+([a-zA-Z0-9_]+)", inc[0])
- for yy in inc3:
- inc2.append (yy)
- return inc2
- else:
- inc = re.findall (ur"^\s*#\s*ifdef\s(.*)", line)
- if len(inc) == 0:
- inc = re.findall (ur"^\s*#\s*ifndef\s(.*)", line)
- if len(inc) > 0:
- inc2 = re.findall ("[A-Za-z_][A-Za-z_0-9]*", inc[0])
- return inc2
- if len(inc) == 0:
- return list ()
- print "WTF. more than one line returned for find_pound_if"
- print inc
- sys.exit(5)
- # IINFO - this is a vector of include information. It consists of 7 elements.
- # [0] - base name of the file
- # [1] - path leading to this file.
- # [2] - orderd list of all headers directly included by this file.
- # [3] - Ordered list of any headers included within condionally compiled code.
- # headers files are expected to have all includes one level deep due to
- # the omnipresent guards at the top of the file.
- # [4] - List of all macros which are consumed (used) within this file.
- # [5] - list of all macros which may be defined in this file.
- # [6] - The source code for this file, if cached.
- # [7] - line number info for any headers in the source file. Indexed by base
- # name, returning the line the include is on.
- empty_iinfo = ("", "", list(), list(), list(), list(), list())
- # This function will process a file and extract interesting information.
- # DO_MACROS indicates whether macros defined and used should be recorded.
- # KEEP_SRC indicates the source for the file should be cached.
- def process_include_info (filen, do_macros, keep_src):
- header = False
- if not os.path.exists (filen):
- return empty_iinfo
- sfile = open (filen, "r");
- data = sfile.readlines()
- sfile.close()
- # Ignore the initial #ifdef HEADER_H in header files
- if filen[-2:] == ".h":
- nest = -1
- header = True
- else:
- nest = 0
- macout = list ()
- macin = list()
- incl = list()
- cond_incl = list()
- src_line = { }
- guard = ""
- for line in (data):
- if is_pound_if (line):
- nest += 1
- elif is_pound_endif (line):
- nest -= 1
- nm = find_pound_include (line, True, True)
- if nm != "" and nm not in incl and nm[-2:] == ".h":
- incl.append (nm)
- if nest > 0:
- cond_incl.append (nm)
- if keep_src:
- src_line[nm] = line
- continue
- if do_macros:
- d = find_pound_define (line)
- if d:
- if d not in macout:
- macout.append (d);
- continue
- d = find_pound_if (line)
- if d:
- # The first #if in a header file should be the guard
- if header and len (d) == 1 and guard == "":
- if d[0][-2:] == "_H":
- guard = d
- else:
- guard = "Guess there was no guard..."
- else:
- for mac in d:
- if mac != "defined" and mac not in macin:
- macin.append (mac);
- if not keep_src:
- data = list()
- return (os.path.basename (filen), os.path.dirname (filen), incl, cond_incl,
- macin, macout, data, src_line)
- # Extract header info, but no macros or source code.
- def process_ii (filen):
- return process_include_info (filen, False, False)
- # Extract header information, and collect macro information.
- def process_ii_macro (filen):
- return process_include_info (filen, True, False)
- # Extract header information, cache the source lines.
- def process_ii_src (filen):
- return process_include_info (filen, False, True)
- # Extract header information, coolewc macro info and cache the source lines.
- def process_ii_macro_src (filen):
- return process_include_info (filen, True, True)
- def ii_base (iinfo):
- return iinfo[0]
- def ii_path (iinfo):
- return iinfo[1]
- def ii_include_list (iinfo):
- return iinfo[2]
- def ii_include_list_cond (iinfo):
- return iinfo[3]
- def ii_include_list_non_cond (iinfo):
- l = ii_include_list (iinfo)
- for n in ii_include_list_cond (iinfo):
- l.remove (n)
- return l
- def ii_macro_consume (iinfo):
- return iinfo[4]
-
- def ii_macro_define (iinfo):
- return iinfo[5]
- def ii_src (iinfo):
- return iinfo[6]
- def ii_src_line (iinfo):
- return iinfo[7]
- def ii_read (fname):
- f = open (fname, 'rb')
- incl = pickle.load (f)
- consumes = pickle.load (f)
- defines = pickle.load (f)
- obj = (fname,fname,incl,list(), list(), consumes, defines, list(), list())
- return obj
- def ii_write (fname, obj):
- f = open (fname, 'wb')
- pickle.dump (obj[2], f)
- pickle.dump (obj[4], f)
- pickle.dump (obj[5], f)
- f.close ()
- # execute a system command which returns file names
- def execute_command (command):
- files = list()
- f = os.popen (command)
- for x in f:
- if x[0:2] == "./":
- fn = x.rstrip()[2:]
- else:
- fn = x.rstrip()
- files.append(fn)
- return files
- # Try to locate a build directory from PATH
- def find_gcc_bld_dir (path):
- blddir = ""
- # Look for blddir/gcc/tm.h
- command = "find " + path + " -mindepth 2 -maxdepth 3 -name tm.h"
- files = execute_command (command)
- for y in files:
- p = os.path.dirname (y)
- if os.path.basename (p) == "gcc":
- blddir = p
- break
- # If not found, try looking a bit deeper
- # Dont look this deep initially because a lot of cross target builds may show
- # up in the list before a native build... but those are better than nothing.
- if not blddir:
- command = "find " + path + " -mindepth 3 -maxdepth 5 -name tm.h"
- files = execute_command (command)
- for y in files:
- p = os.path.dirname (y)
- if os.path.basename (p) == "gcc":
- blddir = p
- break
- return blddir
- # Find files matching pattern NAME, return in a list.
- # CURRENT is True if you want to include the current directory
- # DEEPER is True if you want to search 3 levels below the current directory
- # any files with testsuite diurectories are ignored
- def find_gcc_files (name, current, deeper):
- files = list()
- command = ""
- if current:
- if not deeper:
- command = "find -maxdepth 1 -name " + name + " -not -path \"./testsuite/*\""
- else:
- command = "find -maxdepth 4 -name " + name + " -not -path \"./testsuite/*\""
- else:
- if deeper:
- command = "find -maxdepth 4 -mindepth 2 -name " + name + " -not -path \"./testsuite/*\""
- if command != "":
- files = execute_command (command)
- return files
- # find the list of unique include names found in a file.
- def find_unique_include_list_src (data):
- found = list ()
- for line in data:
- d = find_pound_include (line, True, True)
- if d and d not in found and d[-2:] == ".h":
- found.append (d)
- return found
- # find the list of unique include names found in a file.
- def find_unique_include_list (filen):
- data = open (filen).read().splitlines()
- return find_unique_include_list_src (data)
- # Create the macin, macout, and incl vectors for a file FILEN.
- # macin are the macros that are used in #if* conditional expressions
- # macout are the macros which are #defined
- # incl is the list of incluide files encountered
- # returned as a tuple of the filename followed by the triplet of lists
- # (filen, macin, macout, incl)
- def create_macro_in_out (filen):
- sfile = open (filen, "r");
- data = sfile.readlines()
- sfile.close()
- macout = list ()
- macin = list()
- incl = list()
- for line in (data):
- d = find_pound_define (line)
- if d != "":
- if d not in macout:
- macout.append (d);
- continue
- d = find_pound_if (line)
- if len(d) != 0:
- for mac in d:
- if mac != "defined" and mac not in macin:
- macin.append (mac);
- continue
- nm = find_pound_include (line, True, True)
- if nm != "" and nm not in incl:
- incl.append (nm)
- return (filen, macin, macout, incl)
- # create the macro information for filen, and create .macin, .macout, and .incl
- # files. Return the created macro tuple.
- def create_include_data_files (filen):
- macros = create_macro_in_out (filen)
- depends = macros[1]
- defines = macros[2]
- incls = macros[3]
-
- disp_message = filen
- if len (defines) > 0:
- disp_message = disp_message + " " + str(len (defines)) + " #defines"
- dfile = open (filen + ".macout", "w")
- for x in defines:
- dfile.write (x + "\n")
- dfile.close ()
- if len (depends) > 0:
- disp_message = disp_message + " " + str(len (depends)) + " #if dependencies"
- dfile = open (filen + ".macin", "w")
- for x in depends:
- dfile.write (x + "\n")
- dfile.close ()
- if len (incls) > 0:
- disp_message = disp_message + " " + str(len (incls)) + " #includes"
- dfile = open (filen + ".incl", "w")
- for x in incls:
- dfile.write (x + "\n")
- dfile.close ()
- return macros
- # extract data for include file name_h and enter it into the dictionary.
- # this does not change once read in. use_requires is True if you want to
- # prime the values with already created .requires and .provides files.
- def get_include_data (name_h, use_requires):
- macin = list()
- macout = list()
- incl = list ()
- if use_requires and os.path.exists (name_h + ".requires"):
- macin = open (name_h + ".requires").read().splitlines()
- elif os.path.exists (name_h + ".macin"):
- macin = open (name_h + ".macin").read().splitlines()
- if use_requires and os.path.exists (name_h + ".provides"):
- macout = open (name_h + ".provides").read().splitlines()
- elif os.path.exists (name_h + ".macout"):
- macout = open (name_h + ".macout").read().splitlines()
- if os.path.exists (name_h + ".incl"):
- incl = open (name_h + ".incl").read().splitlines()
- if len(macin) == 0 and len(macout) == 0 and len(incl) == 0:
- return ()
- data = ( name_h, macin, macout, incl )
- return data
-
- # find FIND in src, and replace it with the list of headers in REPLACE.
- # Remove any duplicates of FIND in REPLACE, and if some of the REPLACE
- # headers occur earlier in the include chain, leave them.
- # Return the new SRC only if anything changed.
- def find_replace_include (find, replace, src):
- res = list()
- seen = { }
- anything = False
- for line in src:
- inc = find_pound_include (line, True, True)
- if inc == find:
- for y in replace:
- if seen.get(y) == None:
- res.append("#include \""+y+"\"\n")
- seen[y] = True
- if y != find:
- anything = True
- # if find isnt in the replacement list, then we are deleting FIND, so changes.
- if find not in replace:
- anything = True
- else:
- if inc in replace:
- if seen.get(inc) == None:
- res.append (line)
- seen[inc] = True
- else:
- res.append (line)
- if (anything):
- return res
- else:
- return list()
-
- # pass in a require and provide dictionary to be read in.
- def read_require_provides (require, provide):
- if not os.path.exists ("require-provide.master"):
- print "require-provide.master file is not available. please run data collection."
- sys.exit(1)
- incl_list = open("require-provide.master").read().splitlines()
- for f in incl_list:
- if os.path.exists (f+".requires"):
- require[os.path.basename (f)] = open (f + ".requires").read().splitlines()
- else:
- require[os.path.basename (f)] = list ()
- if os.path.exists (f+".provides"):
- provide[os.path.basename (f)] = open (f + ".provides").read().splitlines()
- else:
- provide [os.path.basename (f)] = list ()
-
- def build_include_list (filen):
- include_files = list()
- sfile = open (filen, "r")
- data = sfile.readlines()
- sfile.close()
- for line in data:
- nm = find_pound_include (line, False, False)
- if nm != "" and nm[-2:] == ".h":
- if nm not in include_files:
- include_files.append(nm)
- return include_files
-
- def build_reverse_include_list (filen):
- include_files = list()
- sfile = open (filen, "r")
- data = sfile.readlines()
- sfile.close()
- for line in reversed(data):
- nm = find_pound_include (line, False, False)
- if nm != "":
- if nm not in include_files:
- include_files.append(nm)
- return include_files
-
- # Get compilation return code, and compensate for a warning that we want to
- # consider an error when it comes to inlined templates.
- def get_make_rc (rc, output):
- rc = rc % 1280
- if rc == 0:
- # This is not considered an error during compilation of an individual file,
- # but it will cause an error during link if it isn't defined. If this
- # warning is seen during compiling a file, make it a build error so we
- # don't remove the header.
- h = re.findall ("warning: inline function.*used but never defined", output)
- if len(h) != 0:
- rc = 1
- return rc;
- def get_make_output (build_dir, make_opt):
- devnull = open('/dev/null', 'w')
- at_a_time = multiprocessing.cpu_count() * 2
- make = "make -j"+str(at_a_time)+ " "
- if build_dir != "":
- command = "cd " + build_dir +"; " + make + make_opt
- else:
- command = make + make_opt
- process = subprocess.Popen(command, stdout=devnull, stderr=subprocess.PIPE, shell=True)
- output = process.communicate();
- rc = get_make_rc (process.returncode, output[1])
- return (rc , output[1])
- def spawn_makes (command_list):
- devnull = open('/dev/null', 'w')
- rc = (0,"", "")
- proc_res = list()
- text = " Trying target builds : "
- for command_pair in command_list:
- tname = command_pair[0]
- command = command_pair[1]
- text += tname + ", "
- c = subprocess.Popen(command, bufsize=-1, stdout=devnull, stderr=subprocess.PIPE, shell=True)
- proc_res.append ((c, tname))
- print text[:-2]
- for p in proc_res:
- output = p[0].communicate()
- ret = (get_make_rc (p[0].returncode, output[1]), output[1], p[1])
- if (ret[0] != 0):
- # Just record the first one.
- if rc[0] == 0:
- rc = ret;
- return rc
- def get_make_output_parallel (targ_list, make_opt, at_a_time):
- command = list()
- targname = list()
- if at_a_time == 0:
- at_a_time = multiprocessing.cpu_count() * 2
- proc_res = [0] * at_a_time
- for x in targ_list:
- if make_opt[-2:] == ".o":
- s = "cd " + x[1] + "/gcc/; make " + make_opt
- else:
- s = "cd " + x[1] +"; make " + make_opt
- command.append ((x[0],s))
- num = len(command)
- rc = (0,"", "")
- loops = num // at_a_time
-
- if (loops > 0):
- for idx in range (loops):
- ret = spawn_makes (command[idx*at_a_time:(idx+1)*at_a_time])
- if ret[0] != 0:
- rc = ret
- break
- if (rc[0] == 0):
- leftover = num % at_a_time
- if (leftover > 0):
- ret = spawn_makes (command[-leftover:])
- if ret[0] != 0:
- rc = ret
- return rc
- def readwholefile (src_file):
- sfile = open (src_file, "r")
- src_data = sfile.readlines()
- sfile.close()
- return src_data
|