Cmdstan 2.29.1 compile error

I run cmdstan within cmdstanr and recently upgraded the cmdstan version to 2.29.1. Now, stan scripts with user defined functions that compiled under 2.28 no longer do. I receive the following message:

template argument deduction/substitution failed:
couldn't deduce template parameter '<anonymous>'

and then error messages like this:

In file included from stan/lib/stan_math/stan/math/rev/core/var.hpp:14,
                 from stan/lib/stan_math/stan/math/rev/meta/conditional_var_value.hpp:4,
                 from stan/lib/stan_math/stan/math/rev/meta.hpp:6,
                 from stan/lib/stan_math/stan/math/rev/core/accumulate_adjoints.hpp:5,
                 from stan/lib/stan_math/stan/math/rev/core.hpp:4,
                 from stan/lib/stan_math/stan/math/rev.hpp:8,
                 from stan/lib/stan_math/stan/math.hpp:19,
                 from stan/src/stan/model/model_header.hpp:4,
                 from C:/Users/SAVITS~1/AppData/Local/Temp/Rtmpiw7wgq/model-6378ff51ca3.hpp:3:
stan/lib/stan_math/stan/math/rev/core/reverse_pass_callback.hpp:38:13: error: 'void stan::math::reverse_pass_callback(F&&) [with F = stan::math::elt_divide(const Mat1&, const Mat2&) [with Mat1 = Eigen::VectorBlock<Eigen::Block<Eigen::Matrix<stan::math::var_value<double>, -1, -1>, -1, 1, true>, -1>; Mat2 = Eigen::Matrix<stan::math::var_value<double>, -1, 1>; stan::require_all_matrix_t<EigMat1, EigMat2>* <anonymous> = 0; stan::require_any_rev_matrix_t<T1, T2>* <anonymous> = 0]::<lambda()>]', declared using local type 'stan::math::elt_divide(const Mat1&, const Mat2&) [with Mat1 = Eigen::VectorBlock<Eigen::Block<Eigen::Matrix<stan::math::var_value<double>, -1, -1>, -1, 1, true>, -1>; Mat2 = Eigen::Matrix<stan::math::var_value<double>, -1, 1>; stan::require_all_matrix_t<EigMat1, EigMat2>* <anonymous> = 0; stan::require_any_rev_matrix_t<T1, T2>* <anonymous> = 0]::<lambda()>', is used but never defined [-fpermissive]
 inline void reverse_pass_callback(F&& functor) {
             ^~~~~~~~~~~~~~~~~~~~~
mingw32-make.exe: *** [make/program:58: C:\Users\SAVITS~1\AppData\Local\Temp\Rtmpiw7wgq\model-6378ff51ca3.exe] Error 1
Error: An error occured during compilation! See the message above for more information.

Are you able to share the Stan model that leads to this error?

I include an example, below, but want to stress that essentially this is just one example of many scripts that successfully compiled and estimated under cmdstan 2.28 that no longer do under 2.29.1.

functions{
  
  vector build_b_spline(vector t, vector ext_knots, int ind, int order);
  matrix build_mux(int N, int start, int end, int K_sp, matrix X, matrix[] G, 
                   matrix beta_x, matrix[] beta_w);
  row_vector build_muxi(int K_sp, int num_basis, row_vector x_i, vector[] g_i, 
                        matrix beta_x, matrix[] beta_w);
  vector build_b_spline(vector t, vector ext_knots, int ind, int order) {
    // INPUTS:
    //    t:          the points at which the b_spline is calculated
    //    ext_knots:  the set of extended knots
    //    ind:        the index of the b_spline
    //    order:      the order of the b-spline
    vector[num_elements(t)] b_spline;
    vector[num_elements(t)] w1 = rep_vector(0, num_elements(t));
    vector[num_elements(t)] w2 = rep_vector(0, num_elements(t));
    if (order==1)
      for (i in 1:num_elements(t)) // B-splines of order 1 are piece-wise constant
        b_spline[i] = (ext_knots[ind] <= t[i]) && (t[i] < ext_knots[ind+1]);
    else {
      if (ext_knots[ind] != ext_knots[ind+order-1])
        w1 = (to_vector(t) - rep_vector(ext_knots[ind], num_elements(t))) /
             (ext_knots[ind+order-1] - ext_knots[ind]);
      if (ext_knots[ind+1] != ext_knots[ind+order])
        w2 = 1 - (to_vector(t) - rep_vector(ext_knots[ind+1], num_elements(t))) /
                 (ext_knots[ind+order] - ext_knots[ind+1]);
      // Calculating the B-spline recursively as linear interpolation of two lower-order splines
      b_spline = w1 .* build_b_spline(t, ext_knots, ind, order-1) +
                 w2 .* build_b_spline(t, ext_knots, ind+1, order-1);
    }
    return b_spline;
  }
  
  matrix build_mux(int N, int start, int end, int K_sp, matrix X, matrix[] G, matrix beta_x, matrix[] beta_w){
    matrix[N,2] mu_x;
    for( arm in 1:2 )
    {
      mu_x[1:N,arm]     = X[start:end,] * to_vector(beta_x[,arm]); /* N x l for each arm */
      // spline term
      for( k in 1:K_sp )
      {
        mu_x[1:N,arm]  += to_vector(beta_w[arm][,k]' * G[k][,start:end]); /* N x 1 */
      } /* end loop k over K predictors */
      
    }/* end loop arm over convenience and reference sample arms */
    
    return mu_x;
  }
  
  row_vector build_muxi(int K_sp, int num_basis,
     row_vector x_i, vector[] g_i, matrix beta_x, matrix[] beta_w){
       
     row_vector[2] mu_xi;
    
     for( arm in 1:2 )
     {
       mu_xi[arm]     = dot_product(x_i,beta_x[,arm]); /* scalar */
       // spline term
       for( k in 1:K_sp )
       {
         mu_xi[arm]   += dot_product(beta_w[arm][1:num_basis,k], g_i[k][1:num_basis]); /* scalar */
       } /* end loop k over K predictors */
      
     }/* end loop arm over convenience and reference sample arms */
    
    return mu_xi;
    
  } /* end function build_mu */
  
  real partial_sum(int[] s, 
                   int start, int end, real[] logit_pw, int K_sp, int n_c, int n,
                   int num_basis, matrix X, matrix[] G, matrix beta_x, matrix[] beta_w, 
                   real phi_w) {
     int N = end - start + 1;
     matrix[N,2] mu_x;
     matrix[N,2] p;
     vector[N] p_tilde; // this pseudoprobability must be in [0,1]
     real fred              = 0;
     
     // memo: slicing on all of mu_x[li,arm], p[li,arm], p_tilde[li] for li in 1:(end-start+1)
     //       where p_tilde is the mean vector for binary data vector, s, and mu_x[,2]
     //       is the mean vector for data vector logit_pw.
     //       Also slicing data vectors s and logit_pw in their respective
     //       log-likelihood contributions.
     
     // for( li in 1:N)
     // {
     //   int i                             = li + start - 1;
     //   vector[num_basis] g_i[K_sp]       = G[1:K_sp,1:num_basis,i];
     //   mu_x[li,]                         = build_muxi(K_sp, num_basis, X[i,], g_i, beta_x, beta_w);
     // }
     mu_x             = build_mux(N, start, end, K_sp, X, G, beta_x, beta_w);
               
     p                = inv_logit( mu_x );
    
    // for( arm in 1:2 )
    // {
    //   for( li in 1:N ) /* 'li' is slicing index */
    //   {
    //     p[li,arm]       = inv_logit( mu_x[li,arm] );
    //   }
    //   
    // } /* end loop i over all units */
    
    // 1. bernoulli likelihood contribution
    p_tilde         = p[1:N,1] ./ ( p[1:N,1] + p[1:N,2] );
    fred            += bernoulli_lpmf( s[1:N] | p_tilde );
    // for( li in 1:N ) // N = end - start + 1, index for slicing
    // {
    //   p_tilde[li]            = p[li,1] ./ ( p[li,1] + p[li,2] );
    //   fred                  += bernoulli_lpmf(s[li] | p_tilde[li]);
    // } /* end loop i over all units */
    
    // 2. normal likelihood contribution
    // In non-threaded model, likelihood statement for n - n_c, logit_pw
    // logit_pw          ~ normal( mu_x[(n_c+1):n,2], phi_w ); 
    // slicing on n length vector mu_x[,2] 
    // subsetting portion of mu_x[,2] linked to logit_pw
    if( start > n_c  ) ## all units in this chunk increment likelihood contribution for logit_pw
    {
      fred       += normal_lpdf( logit_pw[start-n_c:end-n_c] | mu_x[1:N,2], phi_w );
      // for( li in 1:N )
      // {
      //   int i                 = li + start - 1;
      //   fred                  += normal_lpdf( logit_pw[i-n_c] | mu_x[li,2], phi_w );
      // } /* end loop li over n_r units */
    }else{ /* start <= n_c */
      if( end > n_c ) /* some units in chunk < n_c and some > n_c; only those > n_c increment likelihood */
      {
        fred                      += normal_lpdf( logit_pw[1:end-n_c] | mu_x[n_c-start+2:N,2], phi_w );
        
        // for( li in 1:N )
        // {
        //   int i                    = li + start - 1;
        //   if( i - n_c > 0 )
        //      fred                  += normal_lpdf( logit_pw[i-n_c] | mu_x[li,2], phi_w );
        // }
      } /* end if statement on whether n_c \in (start,end ) */
    } /* end if-else statement on whether add logit_pw likelihood contributions */
    
  
  return fred;
  }/* end function partial_sum() */

} /* end function block */

data{
    int<lower=1> n_c; // observed convenience (non-probability) sample size
	  int<lower=1> n_r; // observed reference (probability) sample size
	  int<lower=1> N; // estimate of population size underlying reference and convenience samples
	  int<lower=1> n; // total sample size, n = n_c + n_r
	  int<lower=1> num_cores;
	  int<lower=1> multiplier;
	  int<lower=1> K; // number of fixed effects
	  int<lower=0> K_sp; // number of predictors to model under a spline basis
	  int<lower=1> num_knots;
    int<lower=1> spline_degree;
    matrix[num_knots,K_sp] knots;
    real<lower=0> weights[n_r]; // sampling weights for n_r observed reference sample units
    matrix[n_c, K] X_c; //  *All* predictors - continuous and categorical -  for the convenience units
    matrix[n_r, K] X_r; // *All* predictors - continuous and categorical -  for the reference units
    matrix[n_c, K_sp] Xsp_c; //  *Continuous* predictors under a spline basis for convenience units
    matrix[n_r, K_sp] Xsp_r; // *Continuous* predictors under a spline basis for convenience units
    int<lower=1> n_df;
} /* end data block */

transformed data{
  // create indicator variable of membership in convenience or reference samples
  // indicator of observation membership in the convenience sample
  int grainsize                     = ( n / num_cores ) / multiplier;
  real logit_pw[n_r]                = logit(inv(weights));
  int<lower=0, upper = 1> s[n]      = to_array_1d( append_array(rep_array(1,n_c),rep_array(0,n_r)) ); 
  matrix[n,K] X                     = append_row( X_c,X_r );
  matrix[n,K_sp] X_sp               = append_row( Xsp_c,Xsp_r );
  /* formulate spline basis matrix, B */
  int num_basis = num_knots + spline_degree - 1; // total number of B-splines
  matrix[spline_degree + num_knots,K_sp] ext_knots_temp;
  matrix[2*spline_degree + num_knots,K_sp] ext_knots;
  matrix[num_basis,n] G[K_sp]; /* basis for model on p_c */
  for(k in 1:K_sp)
  {
     ext_knots_temp[,k] = append_row(rep_vector(knots[1,k], spline_degree), knots[,k]);
    // set of extended knots
     ext_knots[,k] = append_row(ext_knots_temp[,k], rep_vector(knots[num_knots,k], spline_degree));
     for (ind in 1:num_basis)
     {
        G[k][ind,] = to_row_vector(build_b_spline(X_sp[,k], ext_knots[,k], ind, (spline_degree + 1)));
     }
     G[k][num_knots + spline_degree - 1, n] = 1;
  }
     
} /* end transformed data block */

parameters {
  matrix<lower=0>[K,2] sigma2_betax; /* standard deviations of K x 2, beta_x */
                                    /* first column is convenience sample, "c", and second column is "r" */

  matrix[K,2] betaraw_x; /* fixed effects coefficients - first colum for p_c; second column for p_r  */
  // spline coefficients
  vector<lower=0>[2] sigma2_global; /* set this equal to 1 if having estimation difficulties */
  matrix<lower=0>[2,K_sp] sigma2_w;
  matrix[num_basis,K_sp] betaraw_w[2];  // vector of B-spline regression coefficients for each predictor, k
                                        // and 2 sample arms
  real<lower=0> phi2_w; /* scale parameter in model for -1og(weights) */                                     
} /* end parameters block */

transformed parameters {
  matrix[K,2] beta_x;
  matrix[num_basis,K_sp] beta_w[2];
  matrix<lower=0>[K,2] sigma_betax;
  vector<lower=0>[2] sigma_global; /* set this equal to 1 if having estimation difficulties */
  matrix<lower=0>[2,K_sp] sigma_w;
  real<lower=0> phi_w;
  
  sigma_betax       = sqrt( sigma2_betax );
  sigma_global      = sqrt( sigma2_global );
  sigma_w           = sqrt( sigma2_w );
  phi_w             = sqrt( phi2_w );
  
  // for scale parameters for interaction effects from those for main effects to which they link
  for( arm in 1:2 )
  {
     beta_x[,arm]   = betaraw_x[,arm] .* sigma_betax[,arm]; /* Non-central parameterization */
  }/* end loop arm over convenience and reference sample arms */
  
  // spline regression coefficients
  for(arm in 1:2)
  {
    for( k in 1:K_sp ) 
    {
      beta_w[arm][,k]   = cumulative_sum(betaraw_w[arm][,k]);
      beta_w[arm][,k]  *= sigma_w[arm,k] * sigma_global[arm];
    } /* end loop k over K predictors */
  }
  
} /* end transformed parameters block */

 model {
  to_vector(sigma2_betax) ~ gamma(1,1);
  to_vector(sigma2_w)     ~ gamma(1,1);
  sigma_global            ~ gamma(1,1);
  phi2_w                  ~ gamma(1,1);

  to_vector(betaraw_x)   ~ std_normal();
  for(arm in 1:2)
    to_vector(betaraw_w[arm])   ~ std_normal();
  
  /* Model likelihood for y, logit_pw */
 // Sum terms 1 to n in the likelihood 
 target            += reduce_sum(partial_sum, s, grainsize, 
                                              logit_pw, K_sp, n_c, n, num_basis, X, G, 
                                              beta_x, beta_w, phi_w); 

} /* end model block */  

generated quantities{
  matrix[n,2] p;
  matrix[n,2] mu_x;
  
  for( arm in 1:2 )
  {
    mu_x[,arm]     = X[,] * to_vector(beta_x[,arm]); /* n x l for each arm */
    // spline term
    for( k in 1:K_sp )
    {
      mu_x[,arm]  += to_vector(beta_w[arm][,k]' * G[k][,1:n]); /* n x 1 */
    } /* end loop k over K predictors */
  
   }/* end loop arm over convenience and reference sample arms */
  
  p        = inv_logit( mu_x );
  
  // smoothed sampling weights for convenience and reference units
  vector[n] weights_smooth_c = inv(p[,1]);
  vector[n] weights_smooth_r = inv(p[,2]);
  // inclusion probabilities in convenience and reference units for convenience units
  // use for soft thresholding
  vector[n_c] pi_c                  = p[1:n_c,1];
  vector[n_c] pi_r_c                = p[1:n_c,2];
  // normalized weights
  weights_smooth_c                  *= ((n_c+0.0)/(n+0.0)) * (sum(weights_smooth_r)/sum(weights_smooth_c));
  weights_smooth_r                  *= ((n_r+0.0)/(n+0.0));
} /* end generated quantities block */

2 Likes

Thanks @tds151! Yes so I think this is a stanc bug. @WardBrian so what’s happening here is that the user defined functions are failing because of the requires for functions like

template <typename T4__, typename T5__, typename T6__, typename T7__,
          stan::require_eigen_matrix_dynamic_t<T4__>*,
          stan::require_stan_scalar_t<T5__>*,
          stan::require_eigen_matrix_dynamic_t<T6__>*,
          stan::require_stan_scalar_t<T7__>*>
  Eigen::Matrix<stan::promote_args_t<stan::value_type_t<T4__>, T5__,
                     stan::value_type_t<T6__>, T7__>, -1, -1>
  build_mux(const int& N, const int& start, const int& end, const int& K_sp,

I think we can just change the * in each function to be * = nullptr and be fine. I can open up a PR for this

Adding *= nullptr will cause Clang to fail if the functions are also provided with forward declarations (we went through this a lot during the development of the overloading PR)

It seems like one issue is that require_col_vector_t doesn’t accept Eigen::Block<Eigen::Map<Eigen::Matrix<double, -1, -1> >, -1, 1, true> as a valid column vector. Adding to_vector around the indexing in the call to build_b_spline fixes this, but we should obviously adjust the template require @stevebronder

I’m not entirely sure what is happening with build_mux

This is the mir I can make to reproduce the error

functions{
  vector build_b_spline(vector t, vector ext_knots, int ind, int order);
  vector build_b_spline(vector t, vector ext_knots, int ind, int order) {
    vector[num_elements(t)] b_spline;
    return b_spline;
  }
} /* end function block */

data{
    vector[10] X_sp;
    vector[10] ext_knots;
} /* end data block */

transformed data{
  vector[10] tester = build_b_spline(X_sp, ext_knots, 1, 5);

} /* end transformed data block */

If you delete the build_b_spline forward decleration at the top everything compiles.

I think we just need to flip where the default values for the additional require template arguments happen. So that for a user defined function we declare the default template values in the forward decleration and not for the function. aka we need to change

template <typename T0__, typename T1__, stan::require_col_vector_t<T0__>*,
          stan::require_col_vector_t<T1__>*>
  Eigen::Matrix<stan::promote_args_t<stan::value_type_t<T0__>, stan::value_type_t<T1__>>, -1, 1>
  build_b_spline(const T0__& t_arg__, const T1__& ext_knots_arg__,
                 const int& ind, const int& order, std::ostream* pstream__) ; 
template <typename T0__, typename T1__,
          stan::require_col_vector_t<T0__>* = nullptr,
          stan::require_col_vector_t<T1__>* = nullptr>
  Eigen::Matrix<stan::promote_args_t<stan::value_type_t<T0__>, stan::value_type_t<T1__>>, -1, 1>
  build_b_spline(const T0__& t_arg__, const T1__& ext_knots_arg__,
                 const int& ind, const int& order, std::ostream* pstream__) {

to

template <typename T0__, typename T1__, stan::require_col_vector_t<T0__>* = nullptr,
          stan::require_col_vector_t<T1__>* = nullptr>
  Eigen::Matrix<stan::promote_args_t<stan::value_type_t<T0__>, stan::value_type_t<T1__>>, -1, 1>
  build_b_spline(const T0__& t_arg__, const T1__& ext_knots_arg__,
                 const int& ind, const int& order, std::ostream* pstream__) ;
template <typename T0__, typename T1__,
          stan::require_col_vector_t<T0__>*,
          stan::require_col_vector_t<T1__>*>
  Eigen::Matrix<stan::promote_args_t<stan::value_type_t<T0__>, stan::value_type_t<T1__>>, -1, 1>
  build_b_spline(const T0__& t_arg__, const T1__& ext_knots_arg__,
                 const int& ind, const int& order, std::ostream* pstream__) {

aka just flipping the = nullptr so that it is done in the forward decl and not for the function with the body

1 Like

Will that work when there is not a forward decl?

For the record, the reason we can’t just do it twice: c++ - C++11: template parameter redefines default argument - Stack Overflow

GCC allows this, even though the spec does not

1 Like

Could we just always write a forward decl before writing the function + body?

I think we could. We already do this for the functors, and it is probably a good idea. If we do that then there’s basically no need to ever have a user write a forward decl, unless they’re using outside C++. We can change the typechecker to allow mutual recursion without them.

It might get a little messy making sure we don’t output them twice if a user separately declared and defined the function.

At any rate @tds151 thank you so much for reporting this and giving us a reproducible example!

We will look into this immediately. If we can fix the issue for this model it might be good to test the other models you mention to see if they’re the same problem or if they need separate fixes, again, if you’re able to share them

Interestingly, this script with different user defined functions successfully compiles:

functions{
  
  // number of non-zero values in a matrix
  int nnz(matrix x) {
    int N = num_elements(x);
    vector[N] x2 = to_vector(x);
    int out = 0;
    for(i in 1:N){
      if(x2[i] != 0) {
        out += 1;
      }
    }
    return(out);
  }
  
  // number of unique values in a vector
  int num_unique(vector x) {
    int nume_x                   = num_elements(x);
    vector[nume_x] x_desc        = sort_desc(x);
    int fred                     = nume_x;
    for(i in 2:nume_x) {
      if( x_desc[i] == x_desc[i-1])
            fred -= 1; 
    } /* end loop over */
   
   return fred;  
  } /* end function num_unique() */
  
  // identify the (first) index where vector x changes value
  int pos_change(int[] x) {
    // vector is in blocks
    int nume_x      = num_elements(x);
    int fred        = 0;
    for (i in 2:nume_x) {
      if( x[i] != x[i-1])
      {
        fred = i;
        break;
      } /* end conditional check on whether current value different from previous */
    } /* end loop over i */
    
    return fred;
  } /* end function num_unique() */
  
  int num_obs(int[] chunk_idx, int[] obs_idx) {
    // observed indices have a "1" in the n*T x 1 obs_idx
    // chunk_idx is the set of indices for which we wish to check if its
    // associated datum value is observed by comparing with the vector
    // of indices whose data values are observed
    int num_check      = num_elements( chunk_idx );
    int fred           = 0;
    for (i in 1:num_check) 
    {
      if( obs_idx[chunk_idx[i]] == 1 )
      {
        fred += 1;
      } /* end conditional check on whether current value different from previous */
    } /* end loop i over number of items in the chunk */
    return fred;
  }/* end function num_matching() */
    
  
  real partial_sum(int[] y_vec, 
                   int start, int end, int[] ind_bin, int[] month, int[] unit, int[] weight_index,
                   vector lambda_y, row_vector[] X, matrix beta_x, vector muvec_z, 
                   vector sigma_lambda, int K, int K_r) {
     int N                    = end - start + 1;
     vector[N] muvec_x;
     /* compute which elements in the slice are observed */
     int num_chunk_obs                = sum(ind_bin[start:end]); /* number of 1s in chunk/slice */
     /* ind_bin[start:end] is an N x 1 vector with indexing starting from 1 */
     matrix[1,N] chunk_bin            = to_matrix(ind_bin[start:end],1,N); /* binary vector */
     /* position of 1's in 1:N */
     int obs_chunk_idx[num_chunk_obs] = csr_extract_v( chunk_bin ); 
     int slice_to_data[num_chunk_obs];
  
     real fred                        = 0;
     
     // construct N x 1, muvec_x and muvec_z
     // since beta_x and beta_z are indexed by month, must compute
     // which units in the slice (chunk) are associated with each month
     // the data vector is stacked month 1 on top of month 2. 
     // so, if there are 2 months in a slice, earlier contiguous indices
     // will be associated with month 1 followed by later contiguous indices
     // associated with month 2
     
     if( month[end] > month[start] ) /* slice contains 2 months */
     { /* month 2 units come after month 1 in indices */
       int pos_newmonth  = pos_change(month[start:end]); /* find position in 1:N of new month */
       int start_t       = 1;
       int end_t         = pos_newmonth-1;
       
       for( t in 1:2 )
       {
          // construct slice subset of Z and X associated to each month
          // on the LHS the indices wills start at 1 and be < N since we are 
          // slicing on muvec_z and muvec_x
          // on the RHS, the indices match the data indices (and will be
          // in start:end)
          for( li in start_t:end_t )  /* index to data */
          {
            int i        = li + start - 1; /* index to slice */
            muvec_x[li]  = X[i] * beta_x[,t];
          } /* end loop k over K fixed effects predictors */
          
          start_t                           = end_t + 1;
          end_t                             = N;
          
       } /* end loop h over the 2 unique months */
     }else{ /* only 1 month in the slice */
          int month_slice                   = month[start];
          for( li in 1:N )  /* index to data */
          {
            int i                           = li + start - 1; /* index to slice */
            muvec_x[li]                     = X[i] * beta_x[,month_slice];
          } /* end loop k over K fixed effects predictors */
     } /* end conditional statement on whether data slice contains 2 months */
     
    
      // 1. latent response likelihood contribution
      fred             += normal_lpdf(lambda_y[start:end] | muvec_x[1:N] + muvec_z[start:end], 
                                     sigma_lambda[weight_index[unit[start:end]]]);
  
      
      // 2. observed data likelihood contribution
      // yobs_vec                ~ poisson_log(lambday_obs);
      // Slicing on both y_vec and lambda_y
      for(li in 1:num_chunk_obs)
            slice_to_data[li] = obs_chunk_idx[li] + start - 1;
      fred             += poisson_log_lpmf( y_vec[obs_chunk_idx] | 
                                            lambda_y[slice_to_data] );
      
  return fred;
  } /* end partial_sum() function  */
} /* end function block */

// The input data is a vector 'y' of length 'N'.
data{
    int<lower=1> n; // number of respondents.  A respondent is a report_with record, split further if necessary to
                    // fit within detailed industry - series code (SS + 6 digit NAICS) and MSA
                    //detailed industry: ESS drop down is the (SS)+(2 Digit Naics), the sub industry is 
                    // approx 4 digit-naics, so in case of '605411' its super sector 60 
                    // and 2 digit naics 54, and its 4 digit naics is 541
    int<lower=1> T; // number of mohths to jointly model y
    int<lower=1> grainsize;
	  int<lower=1> K; // number of linear predictors in marginal model for y
	  int<lower=1> K_r; // number of levels of main and interaction random effects
	  int<lower=0> n_obs; 		// Number of non-missing values
    int<lower=1> ind_obs[n_obs];     	// Vector of non-missing indices in n*T x 1 vector, y
    int<lower=1> n_w; // number of unique weights that will be used to define post-strata
    int<lower=1> n_1; // number of main random effects in K_r = n_1 + n_2 + n_3
    int<lower=1> n_2; // number of second order interaction random effects in K_r
    int<lower=1> n_3; // number of third order interaction random effects in K_r
    int<lower=1> second_order[n_2,2]; // Each row denotes two main effects (\in 1,..n_1)
                                        // attached to each second order random effect
    int<lower=1> third_order[n_3,3]; // Each row denotes three main effects (\in 1,..n_1)
                                        // attached to each third order random effect
    int<lower=0> y[n,T]; // Response variable 
    row_vector[K] X[n*T]; // Predictors include domain-indexed 12 month lagged values,
                    // NO Intercept
                    // Last month's by-record value
                    // Last month's by-domain value
                    // variables used to determine weight stratification, including size class
                    // assume stack 1:n units, each of t = 1,..,T months
     // (w,v,u): w equal to non-zero values in matrix; v = column index of each non-zero value; u = 
    // index of where each row starts in w.
    int<lower=1> num_wv_vals[T];// contains the length of  vector of values !ne 0, w[[t]], under CSR 
                                  // (compressed storage representation.)  
    vector[sum(num_wv_vals)] w_csr; // stacked vector of values in each month, t \in 1,..,T sparse form
    int<lower=1> v_csr[sum(num_wv_vals)]; // column location value for each w value in each month 
                                        // same length as w
    int<lower=1> u_csr[(n+1)*T]; // row locations of w values with padding on either side for each month               
    // Input binary matrix Z in dense matrix format - will be converted to (w,v,u)-sparse representation
    // matrix[n*T, K_r] Z; // Binary matrix of main and interaction random effect links of units
    //                     // Membership or link to random effect levels may change in each month
    //                     // e.g., main effects: detailed industry, MSA, State, Stratum
    //                     // e.g., interaction effects: industry-MSA, industry-State, Industry-Stratum
    //                     // stack n x K_r matrix T times, month-by-month, rbind
    int<lower=1,upper=T> month[n*T]; // month assignment for each i*t case.
    int<lower=0,upper=n> unit[n*T]; // unit assignment for each i*t case
    int<lower=1> weight_index[n]; // post-stratum index for each respondent - takes n_w unique values
                                  // labeling 1,...,n_w must correspond to index of w
    int<lower=1> n_df;
} /* end data block */

transformed data {
  int y_vec[n*T];
  matrix[n*T, K_r] Z; // Binary matrix of main and interaction random effect links of units
  int ind_bin[n*T]; // binary matrix indicating which record is observed
  // int grainsize                     = ( n*T / num_cores ) / multiplier;
  
  // vectorize y for modeling
  for(t in 1:T)
  {
    y_vec[(t-1)*n+1:t*n] = y[1:n,t];// column-major order; stack the T (n x 1) columns
  } /* end loop t over T months */
  
  ind_bin = rep_array(0,n*T);
  for( l in 1:n_obs )
  {
    ind_bin[ind_obs[l]] = 1; // assign position ind_obs[l] to 1 to denote that case is observed
  } /* end loop l over observed cases */

} /* end transformed data block */


// The parameters accepted by the model. 
parameters {
  vector[n*T] lambda_y; /* log-mean of poisson likelihood on y */
  vector<lower=0>[K] sigma_betax; /* standard deviations of K x K covariance matrix n_w, K x 1 rows of beta_x */
  vector<lower=0>[T] sigma_betaz; /* global shrinkage scale of random effects coefficients, K_r x 1, beta_z */
  vector<lower=0>[n_1] lambda_1; /* local scale parameters for first order interaction random effects */ 
  vector<lower=0>[2] delta; /* scale adjustment for second and third order random effects interaction */
                            /* scale. e.g., lambda_2[k] = delta[2] * lambda_1[second_order[k,1],
                                                                     lambda_1[second_order[k,2]]; */
  matrix[K,T] betaraw_x; /* coefficients indexed by stratum (as well as by predictor) */
  matrix[K_r,T] betaraw_z; /* coefficients indexed by stratum (as well as by predictor) */
  //cholesky_factor_corr[K] L_beta; /* cholesky of correlation matrix for K x K Sigma_betay */
  //vector<lower=0>[n_w] sigma_nw; // /* standard deviation in marginal distribution for y */
  vector<lower=0>[n_w] sigma_lambda; // standard deviation of log-mean of y 
} /* end parameters block */


transformed parameters {
  
  matrix[K,T] beta_x;
  matrix[K_r,T] beta_z;
  vector<lower=0>[n_2] lambda_2; /* local scale parameters for second order interaction random effects */
  vector<lower=0>[n_3] lambda_3; /* local scale parameters for third order interaction random effects */
  vector<lower=0>[K_r] lambda; // lambda = rbind( lambda_1, lambda_2, lambda_3 )
  vector[n_obs] lambday_obs;
  vector[n*T] muvec_z;
  
  /* Allow dependence over n_w stratum dimension AND K predictors in fixed effects coefficients for y */
  // place an RW(1) prior on beta_y, beta_w1, beta_w2
  // e.g., beta_y[t] ~ beta_y[t-1] + N_{n_w x K}(Sigma_nw, Sigma_beta)
  for( k in 1:K )
  {
    beta_x[k,] = cumulative_sum(betaraw_x[k,]);
  } /* end loop k over K fixed effects predictors */
  
  for( k in 1:K_r )
  {
    beta_z[k,] = cumulative_sum(betaraw_z[k,]);
  } /* end loop k over K fixed effects predictors */
  
  // for scale parameters for interaction effects from those for main effects to which they link
  lambda_2     = delta[2-1] * (lambda_1[second_order[,1]] .* lambda_1[second_order[,2]]);
  lambda_3     = delta[3-1] * (lambda_1[third_order[,1]]  .* lambda_1[third_order[,2]]
                                                          .* lambda_1[third_order[,3]]);
  
  lambda = append_row(append_row(lambda_1,lambda_2),lambda_3); /* K_r x T */
  
  { /* local block for inducing prior on beta_y */
    for( t in 1:T )
    {
      // beta_y[,t]    =  diag_pre_multiply(sigma_beta,L_beta) *
      //                       beta_y[,t];
      beta_z[,t]   = beta_z[,t] .* (lambda * sigma_betaz[t]);
      beta_x[,t]   = diag_matrix(sigma_betax) * beta_x[,t];
    } /* end loop t over T months to scale rw1 betaint_y */
  } /* end local block */
  
  { /* local block to compute Z_t * beta_z[,t] */
    int start_t = 1;
    // Fill n*T x 1, muvec_z in n x 1 blocks for each month
    for(t in 1:T)
    {
      // extract sparse representation for n x K_r, Z_t from stacked input vectors
      vector[num_wv_vals[t]] Z_tw               = w_csr[start_t:(start_t+num_wv_vals[t]-1)];
      int Z_tv[num_wv_vals[t]]                  = v_csr[start_t:(start_t+num_wv_vals[t]-1)];
      int Z_tu[n+1]                             = u_csr[(t-1)*(n+1)+1:t*(n+1)];
      start_t                                   += num_wv_vals[t];
      // sparse matrix multiply n x K_r, Z_t with K_r x 1, beta_z[,t]
      muvec_z[((t-1)*n + 1):t*n]                = csr_matrix_times_vector(n, K_r, Z_tw, Z_tv, Z_tu, beta_z[,t]); // n x 1 
    } /* end loop t over T months */
  } /* end local block */
  
} /* end transformed parameters block */
  
 model {
  // L_beta          ~ lkj_corr_cholesky(6);
  sigma_betax            ~ student_t(n_df,0,1);
  sigma_betaz            ~ student_t(1,0,1); /* cauchy */
  lambda_1               ~ std_normal();
  delta                  ~ std_normal();
  to_vector(betaraw_x)   ~ std_normal();
  to_vector(betaraw_z)   ~ std_normal();
  
  sigma_lambda           ~ std_normal();
  
  
  /* Model likelihood for lambda_y, y */
 // lambda_y                ~ normal(muvec_y,sigma_lambda[weight_index[unit]]);
 // yobs_vec                ~ poisson_log(lambday_obs);
 target            += reduce_sum(partial_sum, y_vec, grainsize, 
                                              ind_bin, month, unit, weight_index, 
                                              lambda_y, X, beta_x, muvec_z, 
                                              sigma_lambda, K, K_r); 

} /* end model block */  

generated quantities {
  
  vector[n*T] muvec_x;
  vector[n*T] muvec_y;
  matrix[n,T] mumat_y;
  
  matrix[n,T] y_rep; // replicate data for posterior predictive checks
  matrix[n,T] mu_y; // mean of y without over-dispersion parameter - extract this as smoothed y.
  
  for( i in 1:n*T )
  {
    muvec_x[i]   = dot_product(X[i,],to_vector(beta_x[,month[i]]));
  } /* end loop i over n*T respondents */
  
  muvec_y              = muvec_x + muvec_z; // n*T x 1
  mumat_y              = to_matrix(muvec_y,n,T); // column major order, taking n x 1 values
  
  { /* local block for vectorized quantities because Chris demanded it */
    vector[n*T] thetavec_y; // mean of y without over-dispersion parameter.
    vector[n*T] yv_rep;
    for( ell in 1:n*T )
    {
       thetavec_y[ell] = expm1( muvec_y[ell] + 0.5*square(sigma_lambda[weight_index[unit[ell]]]) ) + 1;
    }
    mu_y          = to_matrix(thetavec_y,n,T);
    
    
    for( i in 1:n*T )
    {
      yv_rep[i]        = poisson_log_rng( lambda_y[i] );
    } /* end loop i over n units */

    y_rep   = to_matrix(yv_rep,n,T,1); /* on data scale */
    
  } /* end local block */
  
 
} /* end generated quantities block */

Yes, this error is specifically tied to the user of forward function declarations. Any stan program which doesn’t use those (essentially doesn’t do recursion) should be fine

I have a PR right now that fixes your original model. I’m hoping I can post a link soon to some update stanc3 binaries you could use to test any other models you have issues with

1 Like

Hi @tds151

Available here: Artifacts of stanc3-test-binaries #7 : /bin [Jenkins]

If you are able to download one of these for your correct operating system and place it in cmdstan_path_here/bin named either stanc (non-windows) or stanc.exe (Windows), you should be able to try it out with your models. We’d appreciate it if you confirmed this fixes your issues, and if so this version of stanc will be released shortly as part of a 2.29.2