Hi all,
I’m the developer of the multinma R package, which uses rstan to fit a suite of models for network meta-analysis.
After pushing an update to CRAN this week, I now get failures in gcc-UBSAN additional tests run by CRAN.
Can anyone help me to solve this? I have until 2024-02-08 until my package is threatened with removal. I’ve tried investigating as much as I can (see below), but this is all beyond me!
The full log is here. The first (of many) warnings looks like this:
log output
/data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/CoreEvaluators.h:852:18: runtime error: reference binding to misaligned address 0x63100028148c for type 'struct Scalar', which requires 8 byte alignment
0x63100028148c: note: pointer points here
a0 cb 8f 2d dd 7f 00 00 93 5b 47 76 f3 0a 00 c0 65 c6 8e 5d a0 64 71 40 a0 cb 8f 2d dd 7f 00 00
^
#0 0x7fdd297a5cf8 in Eigen::internal::mapbase_evaluator<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >::coeffRef(long) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/CoreEvaluators.h:852
#1 0x7fdd297a5cf8 in Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> > >, Eigen::internal::evaluator<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> >, 0>::assignCoeff(long) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:637
#2 0x7fdd297a5cf8 in Eigen::internal::dense_assignment_loop<Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> > >, Eigen::internal::evaluator<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> >, 0>, 1, 0>::run(Eigen::internal::generic_dense_assignment_kernel<Eigen::internal::evaluator<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> > >, Eigen::internal::evaluator<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> >, 0>&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:497
#3 0x7fdd297a5cf8 in void Eigen::internal::call_dense_assignment_loop<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > >(Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > const&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:741
#4 0x7fdd297a5cf8 in Eigen::internal::Assignment<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> >, Eigen::internal::Dense2Dense, void>::run(Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > const&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:879
#5 0x7fdd297a5cf8 in void Eigen::internal::call_assignment_no_alias<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > >(Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > const&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:836
#6 0x7fdd297a5cf8 in void Eigen::internal::call_assignment<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > >(Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&, Eigen::internal::assign_op<stan::math::var_value<double, void>, stan::math::var_value<double, void> > const&, Eigen::internal::enable_if<!Eigen::internal::evaluator_assume_aliasing<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, Eigen::internal::evaluator_traits<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >::Shape>::value, void*>::type) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:804
#7 0x7fdd297a5cf8 in void Eigen::internal::call_assignment<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >(Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/AssignEvaluator.h:782
#8 0x7fdd297a5cf8 in Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >& Eigen::MatrixBase<Eigen::Map<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> > >::operator=<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >(Eigen::DenseBase<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> > const&) /data/gannet/ripley/R/test-dev/RcppEigen/include/Eigen/src/Core/Assign.h:66
#9 0x7fdd297a5cf8 in stan::math::arena_matrix<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >& stan::math::arena_matrix<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >::operator=<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >(Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&) /data/gannet/ripley/R/test-dev/StanHeaders/include/stan/math/rev/core/arena_matrix.hpp:128
#10 0x7fdd297a5cf8 in stan::math::arena_matrix<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> >::arena_matrix<Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, (void*)0>(Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&) /data/gannet/ripley/R/test-dev/StanHeaders/include/stan/math/rev/core/arena_matrix.hpp:73
#11 0x7fdd29c3eaa3 in auto stan::math::csr_matrix_times_vector<Eigen::Map<Eigen::Matrix<double, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> >, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1>, (void*)0>(int, int, Eigen::Map<Eigen::Matrix<double, -1, 1, 0, -1, 1>, 0, Eigen::Stride<0, 0> > const&, std::vector<int, std::allocator<int> > const&, std::vector<int, std::allocator<int> > const&, Eigen::Matrix<stan::math::var_value<double, void>, -1, 1, 0, -1, 1> const&) /data/gannet/ripley/R/test-dev/StanHeaders/include/stan/math/rev/fun/csr_matrix_times_vector.hpp:119
#12 0x7fdd294ae606 in stan::scalar_type<std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >, void>::type model_binomial_1par_namespace::model_binomial_1par::log_prob_impl<true, true, std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >, std::vector<int, std::allocator<int> >, (void*)0, (void*)0>(std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&, std::ostream*) const /data/gannet/ripley/R/packages/tests-gcc-SAN/multinma/src/stanExports_binomial_1par.h:1925
#13 0x7fdd294c6f7f in stan::math::var_value<double, void> model_binomial_1par_namespace::model_binomial_1par::log_prob<true, true, stan::math::var_value<double, void> >(std::vector<stan::math::var_value<double, void>, std::allocator<stan::math::var_value<double, void> > >&, std::vector<int, std::allocator<int> >&, std::ostream*) const /data/gannet/ripley/R/packages/tests-gcc-SAN/multinma/src/stanExports_binomial_1par.h:4144
#14 0x7fdd294c6f7f in double stan::model::log_prob_grad<true, true, model_binomial_1par_namespace::model_binomial_1par>(model_binomial_1par_namespace::model_binomial_1par const&, std::vector<double, std::allocator<double> >&, std::vector<int, std::allocator<int> >&, std::vector<double, std::allocator<double> >&, std::ostream*) /data/gannet/ripley/R/test-dev/StanHeaders/include/src/stan/model/log_prob_grad.hpp:40
#15 0x7fdd29e75c78 in std::vector<double, std::allocator<double> > stan::services::util::initialize<true, model_binomial_1par_namespace::model_binomial_1par, stan::io::var_context, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> > >(model_binomial_1par_namespace::model_binomial_1par&, stan::io::var_context const&, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> >&, double, bool, stan::callbacks::logger&, stan::callbacks::writer&) /data/gannet/ripley/R/test-dev/StanHeaders/include/src/stan/services/util/initialize.hpp:167
#16 0x7fdd2a08265b in int stan::services::diagnose::diagnose<model_binomial_1par_namespace::model_binomial_1par>(model_binomial_1par_namespace::model_binomial_1par&, stan::io::var_context const&, unsigned int, unsigned int, double, double, double, stan::callbacks::interrupt&, stan::callbacks::logger&, stan::callbacks::writer&, stan::callbacks::writer&) /data/gannet/ripley/R/test-dev/StanHeaders/include/src/stan/services/diagnose/diagnose.hpp:49
#17 0x7fdd2a08265b in command<model_binomial_1par_namespace::model_binomial_1par, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014, 0, 2147483563>, boost::random::linear_congruential_engine<unsigned int, 40692, 0, 2147483399> > > /data/gannet/ripley/R/test-dev/rstan/include/rstan/stan_fit.hpp:481
#18 0x7fdd2a090ccb in rstan::stan_fit<model_binomial_1par_namespace::model_binomial_1par, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> > >::call_sampler(SEXPREC*) /data/gannet/ripley/R/test-dev/rstan/include/rstan/stan_fit.hpp:1215
#19 0x7fdd294d9f4f in Rcpp::CppMethod1<rstan::stan_fit<model_binomial_1par_namespace::model_binomial_1par, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> > >, SEXPREC*, SEXPREC*>::operator()(rstan::stan_fit<model_binomial_1par_namespace::model_binomial_1par, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> > >*, SEXPREC**) /data/gannet/ripley/R/test-dev/Rcpp/include/Rcpp/module/Module_generated_CppMethod.h:111
#20 0x7fdd29ed7802 in Rcpp::class_<rstan::stan_fit<model_binomial_1par_namespace::model_binomial_1par, boost::random::additive_combine_engine<boost::random::linear_congruential_engine<unsigned int, 40014u, 0u, 2147483563u>, boost::random::linear_congruential_engine<unsigned int, 40692u, 0u, 2147483399u> > > >::invoke_notvoid(SEXPREC*, SEXPREC*, SEXPREC**, int) /data/gannet/ripley/R/test-dev/Rcpp/include/Rcpp/module/class.h:234
#21 0x7fdd67a12caf in CppMethod__invoke_notvoid(SEXPREC*) /tmp/RtmpULoquA/R.INSTALL23bbf97285e75/Rcpp/src/module.cpp:220
#22 0x57baab in do_External /data/gannet/ripley/R/svn/R-devel/src/main/dotcode.c:576
#23 0x685b24 in Rf_eval /data/gannet/ripley/R/svn/R-devel/src/main/eval.c:1248
<< #24 to #279 are similar references to eval.c >>
The other warnings are all in the same model (binomial_1par.stan
), and all similar involving misaligned addresses.
I have managed to recreate the issue using the rocker/r-devel-san docker image, and tracked down the issue to this line in my tests:
expect_warning(nma(smknet, trt_effects = "random", test_grad = TRUE), paste0(m, ".+prior_intercept.+", "prior_trt.+", "prior_het.+"))
For a minimum reproducible example, you can run
library(multinma)
smknet <- set_agd_arm(smoking, studyn, trtc, r = r, n = n, trt_ref = "No intervention")
nma(smknet, trt_effects = "random", test_grad = TRUE)
(test_grad = TRUE
makes no difference, the errors happen with sampling too.)
The sanitizer errors seem to occur the first time that a random effects model (trt_effects = "random"
) is run within a session. Re-running the same model, the errors don’t appear.
Tracking down a little further, I wonder whether this is related to the sparse matrix representation that I am using for the random effects correlation matrix? This is set up in include/transformed_data_common.stan
// Sparse representation
vector[0] wdummy;
array[0] int vudummy;
int RE_L_nz = count_nonzero(RE_L); // Number of non-zero entries
int RE_sparse = RE_L_nz * 1.0 / num_elements(RE_L) <= 0.1; // Use sparse representation? (yes = 1)
vector[RE_sparse ? RE_L_nz : 0] RE_L_w = RE_sparse ? csr_extract_w(RE_L): wdummy; // Non-zero entries
array[RE_sparse ? RE_L_nz : 0] int RE_L_v = RE_sparse ? csr_extract_v(RE_L): vudummy; // v sparse component
array[RE_sparse ? n_delta + 1 : 0] int RE_L_u = RE_sparse ? csr_extract_u(RE_L) : vudummy; // u sparse component
And then applied in include/transformed_parameters_common.stan
to construct the random effects
// -- RE deltas --
// Avoid evaluating tau[1] when no RE (u_delta is zero dim in this case)
vector[n_delta] f_delta =
RE ? (
RE_sparse ?
tau[1] * csr_matrix_times_vector(n_delta, n_delta, RE_L_w, RE_L_v, RE_L_u, u_delta) :
tau[1] * RE_L * u_delta
) : u_delta;
Interestingly, data that doesn’t trigger the sparse representation doesn’t see the sanitizer errors:
library(multinma)
dat <- data.frame(study = rep(LETTERS[1:9], each = 2),
trt = rep(c("a", "b"), times = 9),
r = 1, n = 2)
# Greater than 0.1, not sparse
mean(chol(RE_cor(dat$study, dat$trt, rep(FALSE, nrow(dat)))))
net <- set_agd_arm(dat, study, trt, r = r, n = n)
nma(net, trt_effects = "random", test_grad = TRUE)
whereas data that does trigger the sparse representation does see the sanitizer errors:
library(multinma)
dat <- data.frame(study = rep(LETTERS[1:11], each = 2),
trt = rep(c("a", "b"), times = 11),
r = 1, n = 2)
# Less than 0.1, sparse
mean(chol(RE_cor(dat$study, dat$trt, rep(FALSE, nrow(dat)))))
net <- set_agd_arm(dat, study, trt, r = r, n = n)
nma(net, trt_effects = "random", test_grad = TRUE)
At this point though I’m stuck; I don’t see how to resolve the sanitizer errors beyond blind trial and error. Any help would be hugely appreciated!