Package instant :: Module build
[hide private]
[frames] | no frames]

Source Code for Module instant.build

  1  """This module contains the main part of Instant, the build_module function.""" 
  2   
  3  import os, sys, shutil, glob 
  4  from itertools import chain 
  5   
  6  # TODO: Import only the official interface 
  7  from output import * 
  8  from config import header_and_libs_from_pkgconfig 
  9  from paths import * 
 10  from signatures import * 
 11  from cache import * 
 12  from codegeneration import * 
 13   
 14       
15 -def assert_is_str(x):
16 instant_assert(isinstance(x, str), 17 "In instant.build_module: Expecting string.")
18
19 -def assert_is_bool(x):
20 instant_assert(isinstance(x, bool), 21 "In instant.build_module: Expecting bool.")
22
23 -def assert_is_str_list(x):
24 instant_assert(isinstance(x, (list, tuple)), 25 "In instant.build_module: Expecting sequence.") 26 instant_assert(all(isinstance(i, str) for i in x), 27 "In instant.build_module: Expecting sequence of strings.")
28
29 -def strip_strings(x):
30 assert_is_str_list(x) 31 return [s.strip() for s in x]
32
33 -def arg_strings(x):
34 if isinstance(x, str): 35 x = x.split() 36 return strip_strings(x)
37 38
39 -def copy_files(source, dest, files):
40 """Copy a list of files from a source directory to a destination directory. 41 This may seem a bit complicated, but a lot of this code is error checking.""" 42 if os.path.exists(dest): 43 overwriting = set(files) & set(glob.glob(os.path.join(dest, "*"))) 44 if overwriting: 45 instant_warning("In instant.copy_files: Path '%s' already exists, "\ 46 "overwriting existing files: %r." % (dest, list(overwriting))) 47 else: 48 os.mkdir(dest) 49 50 if source != dest: 51 instant_debug("In instant.copy_files: Copying files %r from %r to %r"\ 52 % (files, source, dest)) 53 54 for f in files: 55 a = os.path.join(source, f) 56 b = os.path.join(dest, f) 57 instant_assert(a != b, "In instant.copy_files: Seems like the "\ 58 "input files are absolute paths, should be relative to "\ 59 "source. (%r, %r)" % (a, b)) 60 instant_assert(os.path.isfile(a), "In instant.copy_files: "\ 61 "Missing source file '%s'." % a) 62 if os.path.isfile(b): 63 os.remove(b) 64 shutil.copyfile(a, b)
65 66
67 -def recompile(modulename, module_path, setup_name, new_compilation_checksum):
68 """Recompile module if the new checksum is different from 69 the one in the checksum file in the module directory.""" 70 # Check if the old checksum matches the new one 71 need_recompilation = True 72 compilation_checksum_filename = "%s.checksum" % modulename 73 if os.path.exists(compilation_checksum_filename): 74 checksum_file = open(compilation_checksum_filename) 75 old_compilation_checksum = checksum_file.readline() 76 checksum_file.close() 77 if old_compilation_checksum == new_compilation_checksum: 78 return 79 80 # Verify that SWIG is on the system 81 (swig_stat, swig_out) = get_status_output("swig -version") 82 if swig_stat != 0: 83 instant_error("In instant.recompile: Could not find swig!"\ 84 " You can download swig from http://www.swig.org") 85 86 # Create log file for logging of compilation errors 87 compile_log_filename = os.path.join(module_path, "compile.log") 88 compile_log_file = open(compile_log_filename, "w") 89 try: 90 # Build module 91 cmd = "python %s build_ext" % setup_name 92 instant_info("--- Instant: compiling ---") 93 instant_debug("cmd = %s" % cmd) 94 ret, output = get_status_output(cmd) 95 compile_log_file.write(output) 96 compile_log_file.flush() 97 if ret != 0: 98 if os.path.exists(compilation_checksum_filename): 99 os.remove(compilation_checksum_filename) 100 instant_error("In instant.recompile: The module did not "\ 101 "compile, see '%s'" % compile_log_filename) 102 103 # 'Install' module 104 cmd = "python %s install --install-platlib=." % setup_name 105 instant_debug("cmd = %s" % cmd) 106 ret, output = get_status_output(cmd) 107 compile_log_file.write(output) 108 compile_log_file.flush() 109 if ret != 0: 110 if os.path.exists(compilation_checksum_filename): 111 os.remove(compilation_checksum_filename) 112 instant_error("In instant.recompile: Could not 'install' "\ 113 "the module, see '%s'" % compile_log_filename) 114 finally: 115 compile_log_file.close() 116 117 # Compilation succeeded, write new_compilation_checksum to checksum_file 118 write_file(compilation_checksum_filename, new_compilation_checksum)
119 120
121 -def copy_to_cache(module_path, cache_dir, modulename):
122 "Copy module directory to cache." 123 # Validate the path 124 cache_module_path = os.path.join(cache_dir, modulename) 125 if os.path.exists(cache_module_path): 126 # TODO: Error instead? Indicates race condition 127 # on disk or bug in Instant. 128 instant_warning("In instant.build_module: Path '%s' already exists,"\ 129 " but module wasn't found in cache previously. Overwriting."\ 130 % cache_module_path) 131 shutil.rmtree(cache_module_path, ignore_errors=True) 132 133 # Error checks 134 instant_assert(os.path.isdir(module_path), "In instant.build_module:"\ 135 " Cannot copy non-existing directory %r!" % module_path) 136 instant_assert(not os.path.isdir(cache_module_path), 137 "In instant.build_module: Cache directory %r shouldn't exist "\ 138 "at this point!" % cache_module_path) 139 instant_debug("In instant.build_module: Copying built module from %r"\ 140 " to cache at %r" % (module_path, cache_module_path)) 141 142 # Do the copying 143 shutil.copytree(module_path, cache_module_path) 144 delete_temp_dir() 145 return cache_module_path
146 147
148 -def build_module(modulename=None, source_directory=".", 149 code="", init_code="", 150 additional_definitions="", additional_declarations="", 151 sources=[], wrap_headers=[], 152 local_headers=[], system_headers=[], 153 include_dirs=['.'], library_dirs=[], libraries=[], 154 swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'], 155 cppargs=['-O2'], lddargs=[], 156 object_files=[], arrays=[], 157 generate_interface=True, generate_setup=True, 158 signature=None, cache_dir=None):
159 """Generate and compile a module from C/C++ code using SWIG. 160 161 Arguments: 162 ========== 163 The keyword arguments are as follows: 164 - B{modulename}: 165 - The name you want for the module. 166 If specified, the module will not be cached. 167 If missing, a name will be constructed based on 168 a checksum of the other arguments, and the module 169 will be placed in the global cache. String. 170 - B{source_directory}: 171 - The directory where used supplied files reside. 172 - B{code}: 173 - A string containing C or C++ code to be compiled and wrapped. 174 - B{init_code}: 175 - Code that should be executed when the instant module is imported. 176 - B{additional_definitions}: 177 - A list of additional definitions (typically needed for inheritance). 178 - B{additional_declarations}: 179 - A list of additional declarations (typically needed for inheritance). 180 - B{sources}: 181 - A list of source files to compile and link with the module. 182 - B{wrap_headers}: 183 - A list of local header files that should be wrapped by SWIG. 184 - B{local_headers}: 185 - A list of local header files required to compile the wrapped code. 186 - B{system_headers}: 187 - A list of system header files required to compile the wrapped code. 188 - B{include_dirs}: 189 - A list of directories to search for header files. 190 - B{library_dirs}: 191 - A list of directories to search for libraries (C{-l}). 192 - B{libraries}: 193 - A list of libraries needed by the instant module. 194 - B{swigargs}: 195 - List of arguments to swig, e.g. C{["-lpointers.i"]} 196 to include the SWIG pointers.i library. 197 - B{cppargs}: 198 - List of arguments to the compiler, e.g. C{["-D", "-U"]}. 199 - B{lddargs}: 200 - List of arguments to the linker, e.g. C{["-D", "-U"]}. 201 - B{object_files}: 202 - If you want to compile the files yourself. TODO: Not yet supported. 203 - B{arrays}: 204 - A list of the C arrays to be made from NumPy arrays. 205 FIXME: Describe this correctly. Tests pass arrays of arrays of strings. 206 - B{generate_interface}: 207 - A bool to indicate if you want to generate the interface files. 208 - B{generate_setup}: 209 - A bool to indicate if you want to generate the setup.py file. 210 - B{signature}: 211 - A signature string to identify the form instead of the source code. 212 - B{cache_dir}: 213 - A directory to look for cached modules and place new ones. 214 If missing, a default directory is used. Note that the module 215 will not be cached if C{modulename} is specified. 216 The cache directory should not be used for anything else. 217 """ 218 219 # Store original directory to be able to restore later 220 original_path = os.getcwd() 221 222 # --- Validate arguments 223 224 instant_assert(modulename is None or isinstance(modulename, str), 225 "In instant.build_module: Expecting modulename to be string or None.") 226 assert_is_str(source_directory) 227 source_directory = os.path.abspath(source_directory) 228 assert_is_str(code) 229 assert_is_str(init_code) 230 assert_is_str(additional_definitions) 231 assert_is_str(additional_declarations) 232 sources = strip_strings(sources) 233 wrap_headers = strip_strings(wrap_headers) 234 local_headers = strip_strings(local_headers) 235 system_headers = strip_strings(system_headers) 236 include_dirs = strip_strings(include_dirs) 237 library_dirs = strip_strings(library_dirs) 238 libraries = strip_strings(libraries) 239 swigargs = arg_strings(swigargs) 240 cppargs = arg_strings(cppargs) 241 lddargs = arg_strings(lddargs) 242 object_files = strip_strings(object_files) 243 arrays = [strip_strings(a) for a in arrays] 244 assert_is_bool(generate_interface) 245 assert_is_bool(generate_setup) 246 instant_assert( signature is None \ 247 or isinstance(signature, str) \ 248 or hasattr(signature, "signature"), 249 "In instant.build_module: Expecting modulename to be string or None.") 250 instant_assert(not (signature is not None and modulename is not None), 251 "In instant.build_module: Can't have both modulename and signature.") 252 253 # --- Replace arguments with defaults if necessary 254 255 cache_dir = validate_cache_dir(cache_dir) 256 257 # Split sources by file-suffix (.c or .cpp) 258 csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')] 259 cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')] 260 if csrcs: 261 instant_error("FIXME: setup.py doesn't use the C sources.") 262 instant_assert(len(csrcs) + len(cppsrcs) == len(sources), 263 "In instant.build_module: Source files must have '.c' or '.cpp' suffix") 264 265 # --- Debugging code 266 instant_debug('In instant.build_module:') 267 instant_debug('::: Begin Arguments :::') 268 instant_debug(' modulename: %r' % modulename) 269 instant_debug(' code: %r' % code) 270 instant_debug(' init_code: %r' % init_code) 271 instant_debug(' additional_definitions: %r' % additional_definitions) 272 instant_debug(' additional_declarations: %r' % additional_declarations) 273 instant_debug(' sources: %r' % sources) 274 instant_debug(' csrcs: %r' % csrcs) 275 instant_debug(' cppsrcs: %r' % cppsrcs) 276 instant_debug(' wrap_headers: %r' % wrap_headers) 277 instant_debug(' local_headers: %r' % local_headers) 278 instant_debug(' system_headers: %r' % system_headers) 279 instant_debug(' include_dirs: %r' % include_dirs) 280 instant_debug(' library_dirs: %r' % library_dirs) 281 instant_debug(' libraries: %r' % libraries) 282 instant_debug(' swigargs: %r' % swigargs) 283 instant_debug(' cppargs: %r' % cppargs) 284 instant_debug(' lddargs: %r' % lddargs) 285 instant_debug(' object_files: %r' % object_files) 286 instant_debug(' arrays: %r' % arrays) 287 instant_debug(' generate_interface: %r' % generate_interface) 288 instant_debug(' generate_setup: %r' % generate_setup) 289 instant_debug(' signature: %r' % signature) 290 instant_debug(' cache_dir: %r' % cache_dir) 291 instant_debug('::: End Arguments :::') 292 293 # --- Setup module directory, making it and copying 294 # files to it if necessary, and compute a modulename 295 # if it isn't specified explicitly 296 297 if modulename is None: 298 # Compute a signature if we have none passed by the user: 299 if signature is None: 300 # Collect arguments used for checksum creation, 301 # including everything that affects the interface 302 # file generation and module compilation. 303 checksum_args = ( \ 304 # We don't care about the modulename, that's what we're trying to construct! 305 #modulename, 306 # We don't care where the user code resides: 307 #source_directory, 308 code, init_code, 309 additional_definitions, 310 additional_declarations, 311 # Skipping filenames, since we use the file contents: 312 #sources, wrap_headers, 313 #local_headers, 314 system_headers, 315 include_dirs, library_dirs, libraries, 316 swigargs, cppargs, lddargs, 317 object_files, arrays, 318 generate_interface, generate_setup, 319 # The signature isn't defined, and the cache_dir doesn't affect the module: 320 #signature, cache_dir) 321 ) 322 allfiles = sources + wrap_headers + local_headers 323 text = "\n".join((str(a) for a in checksum_args)) 324 signature = modulename_from_checksum(compute_checksum(text, allfiles)) 325 modulename = signature 326 moduleids = [signature] 327 else: 328 module, moduleids = check_memory_cache(signature) 329 if module: return module 330 modulename = moduleids[-1] 331 332 # Look for module in disk cache 333 module = check_disk_cache(modulename, cache_dir, moduleids) 334 if module: return module 335 336 # Make a temporary module path for compilation 337 module_path = os.path.join(get_temp_dir(), modulename) 338 instant_assert(not os.path.exists(module_path), 339 "In instant.build_module: Not expecting module_path to exist: '%s'"\ 340 % module_path) 341 os.mkdir(module_path) 342 use_cache = True 343 else: 344 use_cache = False 345 moduleids = [] 346 module_path = os.path.join(original_path, modulename) 347 if not os.path.exists(module_path): 348 os.mkdir(module_path) 349 350 ## Look for module in memory cache 351 #module, moduleids = check_memory_cache(modulename) 352 #if module: return module 353 #instant_assert(modulename == moduleids[-1] and len(moduleids) == 1, "Logic breach.") 354 ## Look for module in local directory 355 #module = check_disk_cache(modulename, original_path, moduleids) 356 #if module: return module 357 358 # Wrapping rest of code in try-block to 359 # clean up at the end if something fails. 360 try: 361 # --- Copy user-supplied files to module path 362 363 module_path = os.path.abspath(module_path) 364 files_to_copy = sources + wrap_headers + local_headers + object_files 365 copy_files(source_directory, module_path, files_to_copy) 366 # At this point, all user input files should reside in module_path. 367 368 # --- Generate additional files in module directory 369 os.chdir(module_path) 370 371 # Generate __init__.py which imports compiled module contents 372 write_file("__init__.py", "from %s import *" % modulename) 373 374 # Generate SWIG interface if wanted 375 ifile_name = "%s.i" % modulename 376 if generate_interface: 377 write_interfacefile(ifile_name, modulename, code, init_code, 378 additional_definitions, additional_declarations, system_headers, 379 local_headers, wrap_headers, arrays) 380 381 # Generate setup.py if wanted 382 setup_name = "setup.py" 383 if generate_setup: 384 write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers, 385 include_dirs, library_dirs, libraries, swigargs, cppargs, lddargs) 386 387 # --- Build module 388 389 # At this point we have all the files, and can make the 390 # total checksum from all file contents. This is used to 391 # decide whether the module needs recompilation or not. 392 393 # Compute new_compilation_checksum 394 # Collect arguments used for checksum creation, 395 # including everything that affects the module compilation. 396 # Since the interface file is included in allfiles, 397 # we don't need stuff that modifies it here. 398 checksum_args = ( \ 399 # We don't care about the modulename, that's what 400 # we're trying to construct! 401 #modulename, 402 # We don't care where the user code resides: 403 #source_directory, 404 #code, init_code, 405 #additional_definitions, additional_declarations, 406 # Skipping filenames, since we use the file contents: 407 #sources, wrap_headers, 408 #local_headers, 409 system_headers, 410 include_dirs, library_dirs, libraries, 411 swigargs, cppargs, lddargs, 412 object_files, #arrays, 413 #generate_interface, generate_setup, 414 # The signature isn't defined, and the 415 # cache_dir doesn't affect the module: 416 #signature, cache_dir) 417 ) 418 text = "\n".join((str(a) for a in checksum_args)) 419 allfiles = sources + wrap_headers + local_headers + [ifile_name] 420 new_compilation_checksum = compute_checksum(text, allfiles) 421 422 # Recompile if necessary 423 recompile(modulename, module_path, setup_name, new_compilation_checksum) 424 425 # --- Load, cache, and return module 426 427 # Copy compiled module to cache 428 if use_cache: 429 module_path = copy_to_cache(module_path, cache_dir, modulename) 430 431 # Import module and place in memory cache 432 module = import_and_cache_module(module_path, modulename, moduleids) 433 if not module: 434 instant_error("Failed to import newly compiled module!") 435 436 instant_debug("In instant.build_module: Returning %s from build_module."\ 437 % module) 438 return module 439 # The end! 440 441 finally: 442 # Always get back to original directory. 443 os.chdir(original_path) 444 445 instant_error("In instant.build_module: Should never reach this point!")
446 # end build_module 447