1r""" 

2 

3*pystencils* automatically searches for a compiler, so in most cases no explicit configuration is required. 

4On Linux make sure that 'gcc' and 'g++' are installed and in your path. 

5On Windows a recent Visual Studio installation is required. 

6In case anything does not work as expected or a special compiler should be used, changes can be specified 

7in a configuration file. 

8 

9*pystencils* looks for a configuration file in JSON format at the following locations in the listed order. 

10 

111. at the path specified in the environment variable ``PYSTENCILS_CONFIG`` 

122. in the current working direction for a file named ``pystencils.json`` 

133. or in your home directory at ``~/.config/pystencils/config.json`` (Linux) or 

14 ``%HOMEPATH%\.pystencils\config.json`` (Windows) 

15 

16If no configuration file is found, a default configuration is created at the above mentioned location in your home. 

17So run *pystencils* once, then edit the created configuration file. 

18 

19 

20Compiler Config (Linux) 

21----------------------- 

22 

23- **'os'**: should be detected automatically as 'linux' 

24- **'command'**: path to C++ compiler (defaults to 'g++') 

25- **'flags'**: space separated list of compiler flags. Make sure to activate OpenMP in your compiler 

26- **'restrict_qualifier'**: the restrict qualifier is not standardized accross compilers. 

27 For most Linux compilers the qualifier is ``__restrict__`` 

28 

29 

30Compiler Config (Windows) 

31------------------------- 

32 

33*pystencils* uses the mechanism of *setuptools.msvc* to search for a compilation environment. 

34Then 'cl.exe' is used to compile. 

35 

36- **'os'**: should be detected automatically as 'windows' 

37- **'msvc_version'**: either a version number, year number, 'auto' or 'latest' for automatic detection of latest 

38 installed version or 'setuptools' for setuptools-based detection. Alternatively path to folder 

39 where Visual Studio is installed. This path has to contain a file called 'vcvarsall.bat' 

40- **'arch'**: 'x86' or 'x64' 

41- **'flags'**: flags passed to 'cl.exe', make sure OpenMP is activated 

42- **'restrict_qualifier'**: the restrict qualifier is not standardized across compilers. 

43 For Windows compilers the qualifier should be ``__restrict`` 

44 

45""" 

46import hashlib 

47import json 

48import os 

49import platform 

50import shutil 

51import subprocess 

52import textwrap 

53from collections import OrderedDict 

54from sysconfig import get_paths 

55from tempfile import TemporaryDirectory, NamedTemporaryFile 

56 

57import numpy as np 

58from appdirs import user_cache_dir, user_config_dir 

59 

60from pystencils import FieldType 

61from pystencils.astnodes import LoopOverCoordinate 

62from pystencils.backends.cbackend import generate_c, get_headers 

63from pystencils.data_types import cast_func, VectorType, vector_memory_access 

64from pystencils.include import get_pystencils_include_path 

65from pystencils.kernel_wrapper import KernelWrapper 

66from pystencils.utils import atomic_file_write, file_handle_for_atomic_write, recursive_dict_update 

67 

68 

69def make_python_function(kernel_function_node, custom_backend=None): 

70 """ 

71 Creates C code from the abstract syntax tree, compiles it and makes it accessible as Python function 

72 

73 The parameters of the kernel are: 

74 - numpy arrays for each field used in the kernel. The keyword argument name is the name of the field 

75 - all symbols which are not defined in the kernel itself are expected as parameters 

76 

77 :param kernel_function_node: the abstract syntax tree 

78 :return: kernel functor 

79 """ 

80 result = compile_and_load(kernel_function_node, custom_backend) 

81 return result 

82 

83 

84def set_config(config): 

85 """ 

86 Override the configuration provided in config file 

87 

88 Configuration of compiler parameters: 

89 If this function is not called the configuration is taken from a config file in JSON format which 

90 is searched in the following locations in the order specified: 

91 - at location provided in environment variable PYSTENCILS_CONFIG (if this variable exists) 

92 - a file called ".pystencils.json" in the current working directory 

93 - ~/.pystencils.json in your home 

94 If none of these files exist a file ~/.pystencils.json is created with a default configuration using 

95 the GNU 'g++' 

96 

97 An example JSON file with all possible keys. If not all keys are specified, default values are used 

98 `` 

99 { 

100 'compiler' : 

101 { 

102 "command": "/software/intel/2017/bin/icpc", 

103 "flags": "-Ofast -DNDEBUG -fPIC -march=native -fopenmp", 

104 "env": { 

105 "LM_PROJECT": "iwia", 

106 } 

107 } 

108 } 

109 `` 

110 """ 

111 global _config 

112 _config = config.copy() 

113 

114 

115def get_configuration_file_path(): 

116 config_path_in_home = os.path.join(user_config_dir('pystencils'), 'config.json') 

117 

118 # 1) Read path from environment variable if found 

119 if 'PYSTENCILS_CONFIG' in os.environ: 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 return os.environ['PYSTENCILS_CONFIG'], True 

121 # 2) Look in current directory for pystencils.json 

122 elif os.path.exists("pystencils.json"): 122 ↛ 123line 122 didn't jump to line 123, because the condition on line 122 was never true

123 return "pystencils.json", True 

124 # 3) Try ~/.pystencils.json 

125 elif os.path.exists(config_path_in_home): 125 ↛ 128line 125 didn't jump to line 128, because the condition on line 125 was never false

126 return config_path_in_home, True 

127 else: 

128 return config_path_in_home, False 

129 

130 

131def create_folder(path, is_file): 

132 if is_file: 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true

133 path = os.path.split(path)[0] 

134 try: 

135 os.makedirs(path) 

136 except os.error: 

137 pass 

138 

139 

140def get_llc_command(): 

141 """Try to get executable for llvm's IR compiler llc 

142 

143 We try if one of the following is in PATH: llc, llc-10, llc-9, llc-8, llc-7, llc-6 

144 """ 

145 candidates = ['llc', 'llc-10', 'llc-9', 'llc-8', 'llc-7', 'llc-6'] 

146 found_executables = (e for e in candidates if shutil.which(e)) 

147 return next(found_executables, None) 

148 

149 

150def read_config(): 

151 if platform.system().lower() == 'linux': 151 ↛ 162line 151 didn't jump to line 162, because the condition on line 151 was never false

152 default_compiler_config = OrderedDict([ 

153 ('os', 'linux'), 

154 ('command', 'g++'), 

155 ('llc_command', get_llc_command() or 'llc'), 

156 ('flags', '-Ofast -DNDEBUG -fPIC -march=native -fopenmp -std=c++11'), 

157 ('restrict_qualifier', '__restrict__') 

158 ]) 

159 if platform.machine().startswith('ppc64'): 159 ↛ 186line 159 didn't jump to line 186, because the condition on line 159 was never false

160 default_compiler_config['flags'] = default_compiler_config['flags'].replace('-march=native', 

161 '-mcpu=native') 

162 elif platform.system().lower() == 'windows': 

163 default_compiler_config = OrderedDict([ 

164 ('os', 'windows'), 

165 ('msvc_version', 'latest'), 

166 ('llc_command', get_llc_command() or 'llc'), 

167 ('arch', 'x64'), 

168 ('flags', '/Ox /fp:fast /OpenMP /arch:avx'), 

169 ('restrict_qualifier', '__restrict') 

170 ]) 

171 elif platform.system().lower() == 'darwin': 

172 default_compiler_config = OrderedDict([ 

173 ('os', 'darwin'), 

174 ('command', 'clang++'), 

175 ('llc_command', get_llc_command() or 'llc'), 

176 ('flags', '-Ofast -DNDEBUG -fPIC -march=native -Xclang -fopenmp -std=c++11'), 

177 ('restrict_qualifier', '__restrict__') 

178 ]) 

179 if platform.machine() == 'arm64': 

180 default_compiler_config['flags'] = default_compiler_config['flags'].replace('-march=native ', '') 

181 for libomp in ['/opt/local/lib/libomp/libomp.dylib', '/usr/local/lib/libomp.dylib', 

182 '/opt/homebrew/lib/libomp.dylib']: 

183 if os.path.exists(libomp): 

184 default_compiler_config['flags'] += ' ' + libomp 

185 break 

186 default_cache_config = OrderedDict([ 

187 ('object_cache', os.path.join(user_cache_dir('pystencils'), 'objectcache')), 

188 ('clear_cache_on_start', False), 

189 ]) 

190 

191 default_config = OrderedDict([('compiler', default_compiler_config), 

192 ('cache', default_cache_config)]) 

193 

194 config_path, config_exists = get_configuration_file_path() 

195 config = default_config.copy() 

196 if config_exists: 196 ↛ 201line 196 didn't jump to line 201, because the condition on line 196 was never false

197 with open(config_path, 'r') as json_config_file: 

198 loaded_config = json.load(json_config_file) 

199 config = recursive_dict_update(config, loaded_config) 

200 else: 

201 create_folder(config_path, True) 

202 with open(config_path, 'w') as f: 

203 json.dump(config, f, indent=4) 

204 

205 if config['cache']['object_cache'] is not False: 205 ↛ 228line 205 didn't jump to line 228, because the condition on line 205 was never false

206 config['cache']['object_cache'] = os.path.expanduser(config['cache']['object_cache']).format(pid=os.getpid()) 

207 

208 clear_cache = False 

209 cache_status_file = os.path.join(config['cache']['object_cache'], 'last_config.json') 

210 if os.path.exists(cache_status_file): 210 ↛ 220line 210 didn't jump to line 220, because the condition on line 210 was never false

211 # check if compiler config has changed 

212 last_config = json.load(open(cache_status_file, 'r')) 

213 if set(last_config.items()) != set(config['compiler'].items()): 213 ↛ 216line 213 didn't jump to line 216, because the condition on line 213 was never false

214 clear_cache = True 

215 else: 

216 for key in last_config.keys(): 

217 if last_config[key] != config['compiler'][key]: 

218 clear_cache = True 

219 

220 if config['cache']['clear_cache_on_start'] or clear_cache: 220 ↛ 223line 220 didn't jump to line 223, because the condition on line 220 was never false

221 shutil.rmtree(config['cache']['object_cache'], ignore_errors=True) 

222 

223 create_folder(config['cache']['object_cache'], False) 

224 with NamedTemporaryFile('w', dir=os.path.dirname(cache_status_file), delete=False) as f: 

225 json.dump(config['compiler'], f, indent=4) 

226 os.replace(f.name, cache_status_file) 

227 

228 if config['compiler']['os'] == 'windows': 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true

229 from pystencils.cpu.msvc_detection import get_environment 

230 msvc_env = get_environment(config['compiler']['msvc_version'], config['compiler']['arch']) 

231 if 'env' not in config['compiler']: 

232 config['compiler']['env'] = {} 

233 config['compiler']['env'].update(msvc_env) 

234 

235 return config 

236 

237 

238_config = read_config() 

239 

240 

241def get_compiler_config(): 

242 return _config['compiler'] 

243 

244 

245def get_cache_config(): 

246 return _config['cache'] 

247 

248 

249def add_or_change_compiler_flags(flags): 

250 if not isinstance(flags, list) and not isinstance(flags, tuple): 

251 flags = [flags] 

252 

253 compiler_config = get_compiler_config() 

254 cache_config = get_cache_config() 

255 cache_config['object_cache'] = False # disable cache 

256 

257 for flag in flags: 

258 flag = flag.strip() 

259 if '=' in flag: 

260 base = flag.split('=')[0].strip() 

261 else: 

262 base = flag 

263 

264 new_flags = [c for c in compiler_config['flags'].split() if not c.startswith(base)] 

265 new_flags.append(flag) 

266 compiler_config['flags'] = ' '.join(new_flags) 

267 

268 

269def clear_cache(): 

270 cache_config = get_cache_config() 

271 if cache_config['object_cache'] is not False: 

272 shutil.rmtree(cache_config['object_cache'], ignore_errors=True) 

273 create_folder(cache_config['object_cache'], False) 

274 

275 

276type_mapping = { 

277 np.float32: ('PyFloat_AsDouble', 'float'), 

278 np.float64: ('PyFloat_AsDouble', 'double'), 

279 np.int16: ('PyLong_AsLong', 'int16_t'), 

280 np.int32: ('PyLong_AsLong', 'int32_t'), 

281 np.int64: ('PyLong_AsLong', 'int64_t'), 

282 np.uint16: ('PyLong_AsUnsignedLong', 'uint16_t'), 

283 np.uint32: ('PyLong_AsUnsignedLong', 'uint32_t'), 

284 np.uint64: ('PyLong_AsUnsignedLong', 'uint64_t'), 

285 np.complex64: (('PyComplex_RealAsDouble', 'PyComplex_ImagAsDouble'), 'ComplexFloat'), 

286 np.complex128: (('PyComplex_RealAsDouble', 'PyComplex_ImagAsDouble'), 'ComplexDouble'), 

287} 

288 

289template_extract_scalar = """ 

290PyObject * obj_{name} = PyDict_GetItemString(kwargs, "{name}"); 

291if( obj_{name} == NULL) {{ PyErr_SetString(PyExc_TypeError, "Keyword argument '{name}' missing"); return NULL; }}; 

292{target_type} {name} = ({target_type}) {extract_function}( obj_{name} ); 

293if( PyErr_Occurred() ) {{ return NULL; }} 

294""" 

295 

296template_extract_complex = """ 

297PyObject * obj_{name} = PyDict_GetItemString(kwargs, "{name}"); 

298if( obj_{name} == NULL) {{ PyErr_SetString(PyExc_TypeError, "Keyword argument '{name}' missing"); return NULL; }}; 

299{target_type} {name}{{ ({real_type}) {extract_function_real}( obj_{name} ), 

300 ({real_type}) {extract_function_imag}( obj_{name} ) }}; 

301if( PyErr_Occurred() ) {{ return NULL; }} 

302""" 

303 

304template_extract_array = """ 

305PyObject * obj_{name} = PyDict_GetItemString(kwargs, "{name}"); 

306if( obj_{name} == NULL) {{ PyErr_SetString(PyExc_TypeError, "Keyword argument '{name}' missing"); return NULL; }}; 

307Py_buffer buffer_{name}; 

308int buffer_{name}_res = PyObject_GetBuffer(obj_{name}, &buffer_{name}, PyBUF_STRIDES | PyBUF_WRITABLE | PyBUF_FORMAT); 

309if (buffer_{name}_res == -1) {{ return NULL; }} 

310""" 

311 

312template_release_buffer = """ 

313PyBuffer_Release(&buffer_{name}); 

314""" 

315 

316template_function_boilerplate = """ 

317static PyObject * {func_name}(PyObject * self, PyObject * args, PyObject * kwargs) 

318{{ 

319 if( !kwargs || !PyDict_Check(kwargs) ) {{  

320 PyErr_SetString(PyExc_TypeError, "No keyword arguments passed");  

321 return NULL;  

322 }} 

323 {pre_call_code} 

324 kernel_{func_name}({parameters}); 

325 {post_call_code} 

326 Py_RETURN_NONE; 

327}} 

328""" 

329 

330template_check_array = """ 

331if(!({cond})) {{  

332 PyErr_SetString(PyExc_ValueError, "Wrong {what} of array {name}. Expected {expected}");  

333 return NULL;  

334}} 

335""" 

336 

337template_size_check = """ 

338if(!({cond})) {{  

339 PyErr_SetString(PyExc_TypeError, "Arrays must have same shape"); return NULL;  

340}}""" 

341 

342template_module_boilerplate = """ 

343static PyMethodDef method_definitions[] = {{ 

344 {method_definitions} 

345 {{NULL, NULL, 0, NULL}} 

346}}; 

347 

348static struct PyModuleDef module_definition = {{ 

349 PyModuleDef_HEAD_INIT, 

350 "{module_name}", /* name of module */ 

351 NULL, /* module documentation, may be NULL */ 

352 -1, /* size of per-interpreter state of the module, 

353 or -1 if the module keeps state in global variables. */ 

354 method_definitions 

355}}; 

356 

357PyMODINIT_FUNC 

358PyInit_{module_name}(void) 

359{{ 

360 return PyModule_Create(&module_definition); 

361}} 

362""" 

363 

364 

365def equal_size_check(fields): 

366 fields = list(fields) 

367 if len(fields) <= 1: 

368 return "" 

369 

370 ref_field = fields[0] 

371 cond = ["(buffer_{field.name}.shape[{i}] == buffer_{ref_field.name}.shape[{i}])".format(ref_field=ref_field, 

372 field=field_to_test, i=i) 

373 for field_to_test in fields[1:] 

374 for i in range(fields[0].spatial_dimensions)] 

375 cond = " && ".join(cond) 

376 return template_size_check.format(cond=cond) 

377 

378 

379def create_function_boilerplate_code(parameter_info, name, ast_node, insert_checks=True): 

380 pre_call_code = "" 

381 parameters = [] 

382 post_call_code = "" 

383 variable_sized_normal_fields = set() 

384 variable_sized_index_fields = set() 

385 

386 for param in parameter_info: 

387 if param.is_field_pointer: 

388 field = param.fields[0] 

389 pre_call_code += template_extract_array.format(name=field.name) 

390 post_call_code += template_release_buffer.format(name=field.name) 

391 parameters.append(f"({str(field.dtype)} *)buffer_{field.name}.buf") 

392 

393 if insert_checks: 393 ↛ 386line 393 didn't jump to line 386, because the condition on line 393 was never false

394 np_dtype = field.dtype.numpy_dtype 

395 item_size = np_dtype.itemsize 

396 

397 aligned = False 

398 if ast_node.assignments: 398 ↛ 403line 398 didn't jump to line 403, because the condition on line 398 was never false

399 aligned = any([a.lhs.args[2] for a in ast_node.assignments 

400 if hasattr(a, 'lhs') and isinstance(a.lhs, cast_func) 

401 and hasattr(a.lhs, 'dtype') and isinstance(a.lhs.dtype, VectorType)]) 

402 

403 if ast_node.instruction_set and aligned: 

404 byte_width = ast_node.instruction_set['width'] * item_size 

405 if 'cachelineZero' in ast_node.instruction_set: 405 ↛ 413line 405 didn't jump to line 413, because the condition on line 405 was never false

406 has_openmp, has_nontemporal = False, False 

407 for loop in ast_node.atoms(LoopOverCoordinate): 

408 has_openmp = has_openmp or any(['#pragma omp' in p for p in loop.prefix_lines]) 

409 has_nontemporal = has_nontemporal or any([a.args[0].field == field and a.args[3] for a in 

410 loop.atoms(vector_memory_access)]) 

411 if has_openmp and has_nontemporal: 

412 byte_width = ast_node.instruction_set['cachelineSize'] 

413 offset = max(max(ast_node.ghost_layers)) * item_size 

414 offset_cond = f"(((uintptr_t) buffer_{field.name}.buf) + {offset}) % {byte_width} == 0" 

415 

416 message = str(offset) + ". This is probably due to a different number of ghost_layers chosen for " \ 

417 "the arrays and the kernel creation. If the number of ghost layers for " \ 

418 "the kernel creation is not specified it will choose a suitable value " \ 

419 "automatically. This value might not " \ 

420 "be compatible with the allocated arrays." 

421 if type(byte_width) is not int: 

422 message += " Note that when both OpenMP and non-temporal stores are enabled, alignment to the "\ 

423 "cacheline size is required." 

424 pre_call_code += template_check_array.format(cond=offset_cond, what="offset", name=field.name, 

425 expected=message) 

426 

427 if (np_dtype.isbuiltin and FieldType.is_generic(field) 

428 and not np.issubdtype(field.dtype.numpy_dtype, np.complexfloating)): 

429 dtype_cond = "buffer_{name}.format[0] == '{format}'".format(name=field.name, 

430 format=field.dtype.numpy_dtype.char) 

431 pre_call_code += template_check_array.format(cond=dtype_cond, what="data type", name=field.name, 

432 expected=str(field.dtype.numpy_dtype)) 

433 

434 item_size_cond = f"buffer_{field.name}.itemsize == {item_size}" 

435 pre_call_code += template_check_array.format(cond=item_size_cond, what="itemsize", name=field.name, 

436 expected=item_size) 

437 

438 if field.has_fixed_shape: 

439 shape_cond = [f"buffer_{field.name}.shape[{i}] == {s}" 

440 for i, s in enumerate(field.spatial_shape)] 

441 shape_cond = " && ".join(shape_cond) 

442 pre_call_code += template_check_array.format(cond=shape_cond, what="shape", name=field.name, 

443 expected=str(field.shape)) 

444 

445 expected_strides = [e * item_size for e in field.spatial_strides] 

446 stride_check_code = "(buffer_{name}.strides[{i}] == {s} || buffer_{name}.shape[{i}]<=1)" 

447 strides_cond = " && ".join([stride_check_code.format(s=s, i=i, name=field.name) 

448 for i, s in enumerate(expected_strides)]) 

449 pre_call_code += template_check_array.format(cond=strides_cond, what="strides", name=field.name, 

450 expected=str(expected_strides)) 

451 else: 

452 if FieldType.is_generic(field): 452 ↛ 454line 452 didn't jump to line 454, because the condition on line 452 was never false

453 variable_sized_normal_fields.add(field) 

454 elif FieldType.is_indexed(field): 

455 variable_sized_index_fields.add(field) 

456 elif param.is_field_stride: 

457 field = param.fields[0] 

458 item_size = field.dtype.numpy_dtype.itemsize 

459 parameters.append("buffer_{name}.strides[{i}] / {bytes}".format(bytes=item_size, i=param.symbol.coordinate, 

460 name=field.name)) 

461 elif param.is_field_shape: 

462 parameters.append(f"buffer_{param.field_name}.shape[{param.symbol.coordinate}]") 

463 else: 

464 extract_function, target_type = type_mapping[param.symbol.dtype.numpy_dtype.type] 

465 if np.issubdtype(param.symbol.dtype.numpy_dtype, np.complexfloating): 465 ↛ 466line 465 didn't jump to line 466, because the condition on line 465 was never true

466 pre_call_code += template_extract_complex.format(extract_function_real=extract_function[0], 

467 extract_function_imag=extract_function[1], 

468 target_type=target_type, 

469 real_type="float" if target_type == "ComplexFloat" 

470 else "double", 

471 name=param.symbol.name) 

472 else: 

473 pre_call_code += template_extract_scalar.format(extract_function=extract_function, 

474 target_type=target_type, 

475 name=param.symbol.name) 

476 

477 parameters.append(param.symbol.name) 

478 

479 pre_call_code += equal_size_check(variable_sized_normal_fields) 

480 pre_call_code += equal_size_check(variable_sized_index_fields) 

481 

482 pre_call_code = textwrap.indent(pre_call_code, ' ') 

483 post_call_code = textwrap.indent(post_call_code, ' ') 

484 return template_function_boilerplate.format(func_name=name, pre_call_code=pre_call_code, 

485 post_call_code=post_call_code, parameters=", ".join(parameters)) 

486 

487 

488def create_module_boilerplate_code(module_name, names): 

489 method_definition = '{{"{name}", (PyCFunction){name}, METH_VARARGS | METH_KEYWORDS, ""}},' 

490 method_definitions = "\n".join([method_definition.format(name=name) for name in names]) 

491 return template_module_boilerplate.format(module_name=module_name, method_definitions=method_definitions) 

492 

493 

494def load_kernel_from_file(module_name, function_name, path): 

495 from importlib.util import spec_from_file_location, module_from_spec 

496 try: 

497 spec = spec_from_file_location(name=module_name, location=path) 

498 mod = module_from_spec(spec) 

499 spec.loader.exec_module(mod) 

500 except ImportError: 

501 import time 

502 import warnings 

503 warnings.warn("Could not load " + path + ", trying on more time...") 

504 time.sleep(1) 

505 spec = spec_from_file_location(name=module_name, location=path) 

506 mod = module_from_spec(spec) 

507 spec.loader.exec_module(mod) 

508 

509 return getattr(mod, function_name) 

510 

511 

512def run_compile_step(command): 

513 compiler_config = get_compiler_config() 

514 config_env = compiler_config['env'] if 'env' in compiler_config else {} 

515 compile_environment = os.environ.copy() 

516 compile_environment.update(config_env) 

517 try: 

518 shell = True if compiler_config['os'].lower() == 'windows' else False 

519 subprocess.check_output(command, env=compile_environment, stderr=subprocess.STDOUT, shell=shell) 

520 except subprocess.CalledProcessError as e: 

521 print(" ".join(command)) 

522 print(e.output.decode('utf8')) 

523 raise e 

524 

525 

526class ExtensionModuleCode: 

527 def __init__(self, module_name='generated', custom_backend=None): 

528 self.module_name = module_name 

529 

530 self._ast_nodes = [] 

531 self._function_names = [] 

532 self._custom_backend = custom_backend 

533 self._code_string = str() 

534 self._code_hash = None 

535 

536 def add_function(self, ast, name=None): 

537 self._ast_nodes.append(ast) 

538 self._function_names.append(name if name is not None else ast.function_name) 

539 

540 def create_code_string(self, restrict_qualifier, function_prefix): 

541 self._code_string = str() 

542 

543 headers = {'<math.h>', '<stdint.h>'} 

544 for ast in self._ast_nodes: 

545 headers.update(get_headers(ast)) 

546 header_list = list(headers) 

547 header_list.sort() 

548 header_list.insert(0, '"Python.h"') 

549 ps_headers = [os.path.join(os.path.dirname(__file__), '..', 'include', h[1:-1]) for h in header_list 

550 if os.path.exists(os.path.join(os.path.dirname(__file__), '..', 'include', h[1:-1]))] 

551 header_hash = b''.join([hashlib.sha256(open(h, 'rb').read()).digest() for h in ps_headers]) 

552 

553 includes = "\n".join(["#include %s" % (include_file,) for include_file in header_list]) 

554 self._code_string += includes 

555 self._code_string += "\n" 

556 self._code_string += f"#define RESTRICT {restrict_qualifier} \n" 

557 self._code_string += f"#define FUNC_PREFIX {function_prefix}" 

558 self._code_string += "\n" 

559 

560 for ast, name in zip(self._ast_nodes, self._function_names): 

561 old_name = ast.function_name 

562 ast.function_name = "kernel_" + name 

563 self._code_string += generate_c(ast, custom_backend=self._custom_backend) 

564 self._code_string += create_function_boilerplate_code(ast.get_parameters(), name, ast) 

565 ast.function_name = old_name 

566 

567 self._code_hash = "mod_" + hashlib.sha256(self._code_string.encode() + header_hash).hexdigest() 

568 self._code_string += create_module_boilerplate_code(self._code_hash, self._function_names) 

569 

570 def get_hash_of_code(self): 

571 assert self._code_string, "The code must be generated first" 

572 return self._code_hash 

573 

574 def write_to_file(self, file): 

575 assert self._code_string, "The code must be generated first" 

576 print(self._code_string, file=file) 

577 

578 

579def compile_module(code, code_hash, base_dir, compile_flags=[]): 

580 compiler_config = get_compiler_config() 

581 extra_flags = ['-I' + get_paths()['include'], '-I' + get_pystencils_include_path()] + compile_flags 

582 

583 if compiler_config['os'].lower() == 'windows': 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true

584 lib_suffix = '.pyd' 

585 object_suffix = '.obj' 

586 windows = True 

587 else: 

588 lib_suffix = '.so' 

589 object_suffix = '.o' 

590 windows = False 

591 

592 src_file = os.path.join(base_dir, code_hash + ".cpp") 

593 lib_file = os.path.join(base_dir, code_hash + lib_suffix) 

594 object_file = os.path.join(base_dir, code_hash + object_suffix) 

595 

596 if not os.path.exists(object_file): 

597 with file_handle_for_atomic_write(src_file) as f: 

598 code.write_to_file(f) 

599 

600 if windows: 600 ↛ 601line 600 didn't jump to line 601, because the condition on line 600 was never true

601 compile_cmd = ['cl.exe', '/c', '/EHsc'] + compiler_config['flags'].split() 

602 compile_cmd += [*extra_flags, src_file, '/Fo' + object_file] 

603 run_compile_step(compile_cmd) 

604 else: 

605 with atomic_file_write(object_file) as file_name: 

606 compile_cmd = [compiler_config['command'], '-c'] + compiler_config['flags'].split() 

607 compile_cmd += [*extra_flags, '-o', file_name, src_file] 

608 run_compile_step(compile_cmd) 

609 

610 # Linking 

611 if windows: 611 ↛ 612line 611 didn't jump to line 612, because the condition on line 611 was never true

612 import sysconfig 

613 config_vars = sysconfig.get_config_vars() 

614 py_lib = os.path.join(config_vars["installed_base"], "libs", 

615 f"python{config_vars['py_version_nodot']}.lib") 

616 run_compile_step(['link.exe', py_lib, '/DLL', '/out:' + lib_file, object_file]) 

617 elif platform.system().lower() == 'darwin': 617 ↛ 618line 617 didn't jump to line 618, because the condition on line 617 was never true

618 with atomic_file_write(lib_file) as file_name: 

619 run_compile_step([compiler_config['command'], '-shared', object_file, '-o', file_name, '-undefined', 

620 'dynamic_lookup'] 

621 + compiler_config['flags'].split()) 

622 else: 

623 with atomic_file_write(lib_file) as file_name: 

624 run_compile_step([compiler_config['command'], '-shared', object_file, '-o', file_name] 

625 + compiler_config['flags'].split()) 

626 return lib_file 

627 

628 

629def compile_and_load(ast, custom_backend=None): 

630 cache_config = get_cache_config() 

631 

632 compiler_config = get_compiler_config() 

633 function_prefix = '__declspec(dllexport)' if compiler_config['os'].lower() == 'windows' else '' 

634 

635 code = ExtensionModuleCode(custom_backend=custom_backend) 

636 code.add_function(ast, ast.function_name) 

637 

638 code.create_code_string(compiler_config['restrict_qualifier'], function_prefix) 

639 code_hash_str = code.get_hash_of_code() 

640 

641 compile_flags = [] 

642 if ast.instruction_set and 'compile_flags' in ast.instruction_set: 642 ↛ 643line 642 didn't jump to line 643, because the condition on line 642 was never true

643 compile_flags = ast.instruction_set['compile_flags'] 

644 

645 if cache_config['object_cache'] is False: 645 ↛ 646line 645 didn't jump to line 646, because the condition on line 645 was never true

646 with TemporaryDirectory() as base_dir: 

647 lib_file = compile_module(code, code_hash_str, base_dir, compile_flags=compile_flags) 

648 result = load_kernel_from_file(code_hash_str, ast.function_name, lib_file) 

649 else: 

650 lib_file = compile_module(code, code_hash_str, base_dir=cache_config['object_cache'], 

651 compile_flags=compile_flags) 

652 result = load_kernel_from_file(code_hash_str, ast.function_name, lib_file) 

653 

654 return KernelWrapper(result, ast.get_parameters(), ast)