PyStan DistutilsExecError/CompileError

I am returning to an old-ish project on a new computer, and I’m having problems compiling a model with PyStan. This is in the context of Mojave 10.14.3, (Anaconda) Python 3.7.2, PyStan 2.18.1, gcc version 4.8.5.

It’s been a while since I used (Py)Stan, so I don’t know if there is something wrong with my model, which is a multilevel logistic regression model with a bunch of intercepts:

data {
  int nobs; // number of observations
  int nreg; // number of regions
  int npat; // number of patients
  int nmat; // number of materials
  int<lower=0,upper=1> y[nobs]; // present = 1, absent = 0
  int p[nobs]; // patient index
  int r[nobs]; // region index
  int m[nobs]; // material index
}
parameters {
  real a_o; // overall intercept
  real a_p[npat]; // patient intercept
  real a_m[nmat]; // material intercepts
  real a_r[nreg]; // region intercept
  real<lower=0.01> sd_p; // patient int sd
}
model {
  // data model
  for(i in 1:nobs)
    y[i] ~ bernoulli(inv_logit(a_o + a_p[p[i]] + a_m[m[i]] + a_r[r[i]]));
  
  // model model
  a_o ~ normal(0,2);
  a_m ~ normal(0,2);
  a_r ~ normal(0,2);
  a_p ~ normal(0,sd_p);
  sd_p ~ gamma(3,2);
}

In case it’s relevant, the data gets processed with this code (all of the *idx columns are 1-indexed, not 0-indexed):

df = pd.read_csv('../data_by_region.csv').dropna()

y = df['pa'].astype(int).values
nobs = len(y)
r = df['ridx'].astype(int).values
nreg = len(df['ridx'].unique())
p = df['pidx'].astype(int).values
npat = len(df['pidx'].unique())
m = df['midx'].astype(int).values
nmat = len(df['midx'].unique())

dd = {'nobs':nobs, 'nreg':nreg, 'npat':npat, 'nmat':nmat, 'y':y, 'p':p, 'r':r, 'm':m}

When I try to run the script to process the data and fit the model, I get this:

INFO:pystan:COMPILING THE C++ CODE FOR MODEL lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29 NOW.
---------------------------------------------------------------------------
DistutilsExecError                        Traceback (most recent call last)
/anaconda3/lib/python3.7/distutils/unixccompiler.py in _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts)
    117             self.spawn(compiler_so + cc_args + [src, '-o', obj] +
--> 118                        extra_postargs)
    119         except DistutilsExecError as msg:

/anaconda3/lib/python3.7/distutils/ccompiler.py in spawn(self, cmd)
    908     def spawn(self, cmd):
--> 909         spawn(cmd, dry_run=self.dry_run)
    910

/anaconda3/lib/python3.7/distutils/spawn.py in spawn(cmd, search_path, verbose, dry_run)
     35     if os.name == 'posix':
---> 36         _spawn_posix(cmd, search_path, dry_run=dry_run)
     37     elif os.name == 'nt':

/anaconda3/lib/python3.7/distutils/spawn.py in _spawn_posix(cmd, search_path, verbose, dry_run)
    158                           "command %r failed with exit status %d"
--> 159                           % (cmd, exit_status))
    160             elif os.WIFSTOPPED(status):

DistutilsExecError: command 'gcc' failed with exit status 1

During handling of the above exception, another exception occurred:

CompileError                              Traceback (most recent call last)
~/Google Drive/analysis/mia/pystan/pystan_swallow_region.py in <module>()
     28
     29 if model_pkl not in pkl_files:
---> 30   model = ps.StanModel(file='lr_swallow_region.stan',model_name='lr_swallow_region')
     31
     32   with open(model_pkl,'wb') as f:

/anaconda3/lib/python3.7/site-packages/pystan/model.py in __init__(self, file, charset, model_name, model_code, stanc_ret, include_paths, boost_lib, eigen_lib, verbose, obfuscate_model_name, extra_compile_args)
    347
    348         try:
--> 349             build_extension.run()
    350         finally:
    351             if redirect_stderr:

/anaconda3/lib/python3.7/distutils/command/build_ext.py in run(self)
    337
    338         # Now actually compile and link everything.
--> 339         self.build_extensions()
    340
    341     def check_extensions_list(self, extensions):

/anaconda3/lib/python3.7/distutils/command/build_ext.py in build_extensions(self)
    446             self._build_extensions_parallel()
    447         else:
--> 448             self._build_extensions_serial()
    449
    450     def _build_extensions_parallel(self):

/anaconda3/lib/python3.7/distutils/command/build_ext.py in _build_extensions_serial(self)
    471         for ext in self.extensions:
    472             with self._filter_build_errors(ext):
--> 473                 self.build_extension(ext)
    474
    475     @contextlib.contextmanager

/anaconda3/lib/python3.7/distutils/command/build_ext.py in build_extension(self, ext)
    531                                          debug=self.debug,
    532                                          extra_postargs=extra_args,
--> 533                                          depends=ext.depends)
    534
    535         # XXX outdated variable, kept here in case third-part code

/anaconda3/lib/python3.7/distutils/ccompiler.py in compile(self, sources, output_dir, macros, include_dirs, debug, extra_preargs, extra_postargs, depends)
    572             except KeyError:
    573                 continue
--> 574             self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
    575
    576         # Return *all* object filenames, not just the ones we just built.

/anaconda3/lib/python3.7/distutils/unixccompiler.py in _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts)
    118                        extra_postargs)
    119         except DistutilsExecError as msg:
--> 120             raise CompileError(msg)
    121
    122     def create_static_lib(self, objects, output_libname,

CompileError: command 'gcc' failed with exit status 1

I can’t just post the data publicly, but I could email it to someone along with the script if that would help diagnose the problem. And I’m happy to share any other relevant information, if I missed anything above. Thanks.

Looks like there’s some sort of compile error. Can you set verbose to
True to try to get the compiler error?

With verbose=True, I get:

INFO:pystan:COMPILING THE C++ CODE FOR MODEL lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29 NOW.
INFO:pystan:OS: darwin, Python: 3.7.2 (default, Dec 29 2018, 00:00:04)
[Clang 4.0.1 (tags/RELEASE_401/final)], Cython 0.28.5
Compiling /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365.pyx because it changed.
[1/1] Cythonizing /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365.pyx
building 'stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365' extension
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders/hy
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T
creating /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd
gcc -Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -I/anaconda3/include -arch x86_64 -I/anaconda3/include -arch x86_64 -DBOOST_RESULT_OF_USE_TR1 -DBOOST_NO_DECLTYPE -DBOOST_DISABLE_ASSERTS -I/var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd -I/anaconda3/lib/python3.7/site-packages/pystan -I/anaconda3/lib/python3.7/site-packages/pystan/stan/src -I/anaconda3/lib/python3.7/site-packages/pystan/stan/lib/stan_math -I/anaconda3/lib/python3.7/site-packages/pystan/stan/lib/stan_math/lib/eigen_3.3.3 -I/anaconda3/lib/python3.7/site-packages/pystan/stan/lib/stan_math/lib/boost_1.66.0 -I/anaconda3/lib/python3.7/site-packages/pystan/stan/lib/stan_math/lib/sundials_3.1.0/include -I/anaconda3/lib/python3.7/site-packages/numpy/core/include -I/anaconda3/include/python3.7m -c /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365.cpp -o /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365.o -O2 -ftemplate-depth-256 -Wno-unused-function -Wno-uninitialized -std=c++11
cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++ [enabled by default]
In file included from /anaconda3/lib/gcc/x86_64-apple-darwin11.4.2/4.8.5/include-fixed/syslimits.h:7:0,
                 from /anaconda3/lib/gcc/x86_64-apple-darwin11.4.2/4.8.5/include-fixed/limits.h:34,
                 from /anaconda3/include/python3.7m/Python.h:11,
                 from /var/folders/hy/4ljd6kf91bj6m8575df3v36m0000gn/T/tmpiugrcknd/stanfit4lr_swallow_region_009d04776976ca2ad4be9a43ddaedb29_7144864738861558365.cpp:57:
/anaconda3/lib/gcc/x86_64-apple-darwin11.4.2/4.8.5/include-fixed/limits.h:168:61: fatal error: limits.h: No such file or directory
 #include_next <limits.h>  /* recurse down to the real one */
                                                             ^
compilation terminated.
ERROR:root:Internal Python error in the inspect module.
Below is the traceback from this internal error.

Traceback (most recent call last):
  File "/anaconda3/lib/python3.7/distutils/unixccompiler.py", line 118, in _compile
    extra_postargs)
  File "/anaconda3/lib/python3.7/distutils/ccompiler.py", line 909, in spawn
    spawn(cmd, dry_run=self.dry_run)
  File "/anaconda3/lib/python3.7/distutils/spawn.py", line 36, in spawn
    _spawn_posix(cmd, search_path, dry_run=dry_run)
  File "/anaconda3/lib/python3.7/distutils/spawn.py", line 159, in _spawn_posix
    % (cmd, exit_status))
distutils.errors.DistutilsExecError: command 'gcc' failed with exit status 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 2961, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-4-a5e426ed2e0b>", line 1, in <module>
    model = ps.StanModel(file='lr_swallow_region.stan',model_name='lr_swallow_region', verbose=True)
  File "/anaconda3/lib/python3.7/site-packages/pystan/model.py", line 349, in __init__
    build_extension.run()
  File "/anaconda3/lib/python3.7/distutils/command/build_ext.py", line 339, in run
    self.build_extensions()
  File "/anaconda3/lib/python3.7/distutils/command/build_ext.py", line 448, in build_extensions
    self._build_extensions_serial()
  File "/anaconda3/lib/python3.7/distutils/command/build_ext.py", line 473, in _build_extensions_serial
    self.build_extension(ext)
  File "/anaconda3/lib/python3.7/distutils/command/build_ext.py", line 533, in build_extension
    depends=ext.depends)
  File "/anaconda3/lib/python3.7/distutils/ccompiler.py", line 574, in compile
    self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
  File "/anaconda3/lib/python3.7/distutils/unixccompiler.py", line 120, in _compile
    raise CompileError(msg)
distutils.errors.CompileError: command 'gcc' failed with exit status 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/anaconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 1863, in showtraceback
    stb = value._render_traceback_()
AttributeError: 'CompileError' object has no attribute '_render_traceback_'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py", line 1095, in get_records
    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
  File "/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py", line 311, in wrapped
    return f(*args, **kwargs)
  File "/anaconda3/lib/python3.7/site-packages/IPython/core/ultratb.py", line 345, in _fixed_getinnerframes
    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
  File "/anaconda3/lib/python3.7/inspect.py", line 1502, in getinnerframes
    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)
  File "/anaconda3/lib/python3.7/inspect.py", line 1460, in getframeinfo
    filename = getsourcefile(frame) or getfile(frame)
  File "/anaconda3/lib/python3.7/inspect.py", line 696, in getsourcefile
    if getattr(getmodule(object, filename), '__loader__', None) is not None:
  File "/anaconda3/lib/python3.7/inspect.py", line 742, in getmodule
    os.path.realpath(f)] = module.__name__
AttributeError: module has no attribute '__name__'
INFO:root:
Unfortunately, your original traceback can not be constructed.

I think this is the same problem as other Mojave Users have had

#include_next <limits.h>

Try this based on the thread https://github.com/stan-dev/pystan/issues/521

open /Library/Developer/CommandLineTools/Packages/macOS_SDK_headers_for_macOS_10.14.pkg

That worked. Thanks!